You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2021/05/25 00:27:03 UTC
[incubator-pinot] branch master updated: SegmentDirectoryLoader
interface and local impl (#6949)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2b55004 SegmentDirectoryLoader interface and local impl (#6949)
2b55004 is described below
commit 2b5500447cd82cedca9dd47860e69d321c4e00c3
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Mon May 24 17:26:42 2021 -0700
SegmentDirectoryLoader interface and local impl (#6949)
Introducing a SegmentDirectoryLoader which will create the SegmentDirectory, with default implementation being SegmentLocalFSDirectory.
---
.../broker/pruner/PartitionZKMetadataPruner.java | 4 +-
.../segmentpruner/PartitionSegmentPruner.java | 4 +-
.../broker/pruner/SegmentZKMetadataPrunerTest.java | 2 +-
.../routing/segmentpruner/SegmentPrunerTest.java | 2 +-
.../metadata/segment/SegmentPartitionMetadata.java | 1 +
.../apache/pinot/common/utils/SegmentUtils.java | 2 +-
.../common/metadata/SegmentZKMetadataTest.java | 2 +-
.../segment/ColumnPartitionMetadataTest.java | 1 +
.../resources/LLCSegmentCompletionHandlers.java | 4 +-
.../segment/OfflineSegmentAssignment.java | 2 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 6 +-
.../segment/CommittingSegmentDescriptor.java | 2 +-
.../helix/core/util/ZKMetadataUtils.java | 6 +-
.../realtime/provisioning/MemoryEstimator.java | 2 +-
.../OfflineReplicaGroupSegmentAssignmentTest.java | 2 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 2 +-
.../controller/utils/SegmentMetadataMockUtils.java | 6 +-
.../realtime/LLRealtimeSegmentDataManager.java | 6 +-
.../core/metadata/DefaultMetadataExtractor.java | 2 +-
.../pinot/core/minion/RawIndexConverter.java | 6 +-
.../apache/pinot/core/minion/SegmentConverter.java | 2 +-
.../apache/pinot/core/minion/SegmentPurger.java | 2 +-
.../minion/rollup/MergeRollupSegmentConverter.java | 2 +-
.../processing/collector/ConcatCollector.java | 2 +-
.../partitioner/TableConfigPartitioner.java | 2 +-
.../pinot/core/util/GenericRowSerDeUtils.java | 2 +-
.../pinot/core/minion/SegmentPurgerTest.java | 17 ++--
.../query/pruner/ColumnValueSegmentPrunerTest.java | 2 +-
.../framework/SegmentProcessingFrameworkTest.java | 2 +-
.../converter/RealtimeSegmentConverterTest.java | 2 +-
...vertToRawIndexMinionClusterIntegrationTest.java | 2 +-
...PartitionLLCRealtimeClusterIntegrationTest.java | 2 +-
.../pinot/perf/BenchmarkFixedBitIntReader.java | 2 +-
.../BenchmarkFixedBitSVForwardIndexReader.java | 2 +-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 18 ++--
.../pinot/perf/BenchmarkPinotDataBitSet.java | 2 +-
.../pinot/perf/BenchmarkPinotDataBuffer.java | 2 +-
.../apache/pinot/perf/BenchmarkQueryEngine.java | 2 +-
.../org/apache/pinot/perf/RawIndexBenchmark.java | 2 +-
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 2 +-
.../job/partitioners/PartitionFunctionFactory.java | 8 +-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 4 +-
.../filebased/FileBasedSegmentWriterTest.java | 2 +-
.../indexsegment/immutable/EmptyIndexSegment.java | 4 +-
.../immutable/ImmutableSegmentImpl.java | 6 +-
.../immutable/ImmutableSegmentLoader.java | 49 ++++++----
.../indexsegment/mutable/IntermediateSegment.java | 4 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 14 ++-
.../local/io/reader/impl/FixedBitIntReader.java | 2 +-
.../impl/FixedByteSingleValueMultiColReader.java | 2 +-
.../readerwriter/PinotDataBufferMemoryManager.java | 2 +-
.../RealtimeIndexOffHeapMemoryManager.java | 2 +-
...ixedByteSingleValueMultiColumnReaderWriter.java | 2 +-
.../local/io/util/FixedBitIntReaderWriter.java | 2 +-
.../local/io/util/FixedBitIntReaderWriterV2.java | 4 +-
.../local/io/util/FixedByteValueReaderWriter.java | 2 +-
.../segment/local/io/util/PinotDataBitSet.java | 2 +-
.../segment/local/io/util/PinotDataBitSetV2.java | 4 +-
.../local/io/util/VarLengthValueReader.java | 2 +-
.../local/io/util/VarLengthValueWriter.java | 2 +-
.../local/io/writer/impl/DirectMemoryManager.java | 2 +-
.../writer/impl/FixedBitMVForwardIndexWriter.java | 2 +-
.../writer/impl/FixedBitSVForwardIndexWriter.java | 2 +-
.../impl/FixedByteSingleValueMultiColWriter.java | 2 +-
.../local/io/writer/impl/MmapMemoryManager.java | 2 +-
.../writer/impl/MutableOffHeapByteArrayStore.java | 2 +-
.../local/loader/LocalSegmentDirectoryLoader.java | 50 +++++++++++
.../dictionary/BaseOffHeapMutableDictionary.java | 2 +-
.../impl/dictionary/OffHeapMutableBytesStore.java | 2 +-
.../forward/FixedByteMVMutableForwardIndex.java | 2 +-
.../forward/FixedByteSVMutableForwardIndex.java | 2 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 7 +-
.../creator/impl/SegmentDictionaryCreator.java | 3 +-
.../impl/SegmentIndexCreationDriverImpl.java | 3 +-
.../impl/bloom/OnHeapGuavaBloomFilterCreator.java | 2 +-
.../fwd/MultiValueUnsortedForwardIndexCreator.java | 2 +-
.../fwd/SingleValueFixedByteRawIndexCreator.java | 2 +-
.../fwd/SingleValueSortedForwardIndexCreator.java | 2 +-
.../SingleValueUnsortedForwardIndexCreator.java | 2 +-
.../fwd/SingleValueVarByteRawIndexCreator.java | 2 +-
.../impl/inv/BitmapInvertedIndexWriter.java | 2 +-
.../inv/OffHeapBitmapInvertedIndexCreator.java | 4 +-
.../impl/inv/OnHeapBitmapInvertedIndexCreator.java | 2 +-
.../creator/impl/inv/RangeIndexCreator.java | 4 +-
.../impl/inv/geospatial/BaseH3IndexCreator.java | 2 +-
.../impl/inv/geospatial/OffHeapH3IndexCreator.java | 2 +-
.../impl/inv/json/BaseJsonIndexCreator.java | 4 +-
.../impl/inv/json/OffHeapJsonIndexCreator.java | 2 +-
.../impl/inv/text/LuceneFSTIndexCreator.java | 2 +-
.../impl/nullvalue/NullValueVectorCreator.java | 2 +-
.../stats/AbstractColumnStatisticsCollector.java | 2 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 4 +-
.../index/column/BaseVirtualColumnProvider.java | 2 +-
.../DefaultNullValueVirtualColumnProvider.java | 2 +-
.../index/column/PhysicalColumnIndexContainer.java | 8 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 35 +++++---
.../segment/index/datasource/EmptyDataSource.java | 2 +-
.../index/datasource/ImmutableDataSource.java | 2 +-
.../segment/index/loader/IndexLoadingConfig.java | 27 +++++-
.../local/segment/index/loader/LoaderUtils.java | 8 +-
.../segment/index/loader/SegmentPreProcessor.java | 18 ++--
.../loader/bloomfilter/BloomFilterHandler.java | 12 +--
.../ColumnMinMaxValueGenerator.java | 10 +--
.../defaultcolumn/BaseDefaultColumnHandler.java | 10 +--
.../defaultcolumn/DefaultColumnHandlerFactory.java | 4 +-
.../defaultcolumn/V1DefaultColumnHandler.java | 4 +-
.../defaultcolumn/V3DefaultColumnHandler.java | 8 +-
.../index/loader/invertedindex/H3IndexHandler.java | 10 +--
.../loader/invertedindex/InvertedIndexHandler.java | 10 +--
.../loader/invertedindex/JsonIndexHandler.java | 10 +--
.../invertedindex/LuceneFSTIndexHandler.java | 10 +--
.../loader/invertedindex/RangeIndexHandler.java | 10 +--
.../loader/invertedindex/TextIndexHandler.java | 16 ++--
.../index/readers/BaseImmutableDictionary.java | 2 +-
.../index/readers/BitmapInvertedIndexReader.java | 2 +-
.../segment/index/readers/BytesDictionary.java | 2 +-
.../segment/index/readers/DoubleDictionary.java | 2 +-
.../segment/index/readers/FloatDictionary.java | 2 +-
.../local/segment/index/readers/IntDictionary.java | 2 +-
.../segment/index/readers/LongDictionary.java | 2 +-
.../index/readers/LuceneFSTIndexReader.java | 2 +-
.../index/readers/NullValueVectorReaderImpl.java | 2 +-
.../segment/index/readers/OnHeapDictionary.java | 2 +-
.../index/readers/OnHeapDoubleDictionary.java | 2 +-
.../index/readers/OnHeapFloatDictionary.java | 2 +-
.../segment/index/readers/OnHeapIntDictionary.java | 2 +-
.../index/readers/OnHeapLongDictionary.java | 2 +-
.../index/readers/OnHeapStringDictionary.java | 2 +-
.../segment/index/readers/RangeIndexReader.java | 2 +-
.../segment/index/readers/StringDictionary.java | 2 +-
.../readers/bloom/BaseGuavaBloomFilterReader.java | 2 +-
.../readers/bloom/BloomFilterReaderFactory.java | 2 +-
.../bloom/OffHeapGuavaBloomFilterReader.java | 2 +-
.../bloom/OnHeapGuavaBloomFilterReader.java | 2 +-
.../forward/BaseChunkSVForwardIndexReader.java | 4 +-
.../forward/FixedBitMVForwardIndexReader.java | 2 +-
.../forward/FixedBitSVForwardIndexReader.java | 2 +-
.../forward/FixedBitSVForwardIndexReaderV2.java | 2 +-
.../FixedByteChunkSVForwardIndexReader.java | 2 +-
.../forward/VarByteChunkSVForwardIndexReader.java | 2 +-
.../readers/geospatial/ImmutableH3IndexReader.java | 2 +-
.../readers/json/ImmutableJsonIndexReader.java | 2 +-
.../readers/sorted/SortedIndexReaderImpl.java | 2 +-
.../index/readers/text/LuceneTextIndexReader.java | 9 +-
.../local/segment/store/FilePerIndexDirectory.java | 11 +--
.../segment/local/segment/store/IndexEntry.java | 2 +-
.../segment/local/segment/store/IndexKey.java | 1 +
.../segment/store/SegmentLocalFSDirectory.java | 92 ++++++++++---------
.../segment/store/SingleFileIndexDirectory.java | 10 ++-
.../virtualcolumn/DocIdVirtualColumnProvider.java | 2 +-
.../virtualcolumn/VirtualColumnProvider.java | 2 +-
.../segment/local/startree/OffHeapStarTree.java | 2 +-
.../local/startree/OffHeapStarTreeNode.java | 2 +-
.../local/startree/StarTreeBuilderUtils.java | 8 +-
.../startree/v2/builder/MultipleTreesBuilder.java | 6 +-
.../v2/builder/OffHeapSingleTreeBuilder.java | 2 +-
.../startree/v2/builder/StarTreeIndexCombiner.java | 2 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 4 +-
.../startree/v2/store/StarTreeIndexContainer.java | 4 +-
.../startree/v2/store/StarTreeLoaderUtils.java | 6 +-
.../apache/pinot/segment/local/utils/CrcUtils.java | 2 +-
.../segment/local/utils/SegmentPushUtils.java | 2 +-
.../local/utils/fst/PinotBufferIndexInput.java | 2 +-
.../io/reader/impl/FixedBitIntReaderTest.java | 2 +-
.../segment/local/io/util/PinotDataBitSetTest.java | 2 +-
.../local/io/util/PinotDataBitSetV2Test.java | 2 +-
.../io/util/VarLengthValueReaderWriterTest.java | 2 +-
.../io/writer/impl/MmapMemoryManagerTest.java | 2 +-
.../creator/BitmapInvertedIndexCreatorTest.java | 4 +-
.../local/segment/creator/IntArraysTest.java | 4 +-
.../impl/SegmentColumnarIndexCreatorTest.java | 1 +
.../local/segment/index/ColumnMetadataTest.java | 4 +-
.../segment/local/segment/index/H3IndexTest.java | 4 +-
.../segment/local/segment/index/JsonIndexTest.java | 4 +-
.../segment/index/SegmentMetadataImplTest.java | 4 +-
.../DefaultNullValueVirtualColumnProviderTest.java | 2 +-
.../SegmentV1V2ToV3FormatConverterTest.java | 6 +-
.../index/creator/BloomFilterCreatorTest.java | 4 +-
.../index/creator/LuceneFSTIndexCreatorTest.java | 4 +-
.../index/creator/RangeIndexCreatorTest.java | 4 +-
.../segment/index/creator/RawIndexCreatorTest.java | 15 +++-
.../SegmentGenerationWithBytesTypeTest.java | 10 ++-
.../SegmentGenerationWithFilterRecordsTest.java | 5 +-
.../SegmentGenerationWithMultipleRecordsTest.java | 5 +-
.../SegmentGenerationWithNoRecordsTest.java | 5 +-
.../SegmentGenerationWithTimeColumnTest.java | 11 ++-
.../index/creator/SegmentPartitionTest.java | 6 +-
.../index/forward/FixedBitMVForwardIndexTest.java | 4 +-
.../forward/FixedByteChunkSVForwardIndexTest.java | 2 +-
.../forward/SortedForwardIndexReaderTest.java | 2 +-
.../forward/VarByteChunkSVForwardIndexTest.java | 6 +-
.../local/segment/index/loader/LoaderTest.java | 67 ++++++++------
.../index/loader/SegmentPreProcessorTest.java | 100 ++++++++++++++++-----
.../BaseDefaultColumnHandlerTest.java | 7 +-
.../FixedByteWidthRowColDataFileReaderTest.java | 2 +-
.../index/readers/ImmutableDictionaryTest.java | 4 +-
.../ImmutableDictionaryTypeConversionTest.java | 4 +-
.../readers/NullValueVectorReaderImplTest.java | 2 +-
.../FixedBitSVForwardIndexReaderV2Test.java | 2 +-
...FixedByteWidthRowColForwardIndexWriterTest.java | 4 +-
.../store/ColumnIndexDirectoryTestHelper.java | 6 +-
.../segment/store/FilePerIndexDirectoryTest.java | 6 +-
.../segment/store/SegmentDirectoryPathsTest.java | 3 +-
.../segment/store/SegmentLocalFSDirectoryTest.java | 35 ++++----
.../store/SingleFileIndexDirectoryTest.java | 6 +-
.../v2/builder/StarTreeV2BuilderConfigTest.java | 4 +-
.../segment/local/utils/fst/FSTBuilderTest.java | 2 +-
pinot-segment-spi/pom.xml | 4 +
.../org/apache/pinot/segment/spi}/V1Constants.java | 4 +-
.../segment/spi/index}/creator/TextIndexType.java | 2 +-
.../spi}/index/metadata/ColumnMetadata.java | 10 +--
.../spi}/index/metadata/SegmentMetadataImpl.java | 27 +++---
.../segment/spi/loader/SegmentDirectoryLoader.java | 52 ++++-------
.../spi/loader/SegmentDirectoryLoaderRegistry.java | 89 ++++++++++++++++++
.../SegmentLoader.java} | 33 ++++---
.../segment/spi}/memory/BasePinotLBuffer.java | 2 +-
.../pinot/segment/spi/memory}/CleanerUtil.java | 4 +-
.../pinot/segment/spi}/memory/PinotByteBuffer.java | 3 +-
.../pinot/segment/spi}/memory/PinotDataBuffer.java | 2 +-
.../spi}/memory/PinotNativeOrderLBuffer.java | 2 +-
.../spi}/memory/PinotNonNativeOrderLBuffer.java | 2 +-
.../spi}/partition/ByteArrayPartitionFunction.java | 3 +-
.../spi}/partition/HashCodePartitionFunction.java | 3 +-
.../spi}/partition/ModuloPartitionFunction.java | 3 +-
.../spi}/partition/MurmurPartitionFunction.java | 48 ++++++++--
.../segment/spi/partition/PartitionFunction.java | 1 -
.../spi}/partition/PartitionFunctionFactory.java | 3 +-
.../metadata}/ColumnPartitionMetadata.java | 2 +-
.../segment/spi}/store/ColumnIndexDirectory.java | 12 ++-
.../pinot/segment/spi}/store/ColumnIndexType.java | 2 +-
.../pinot/segment/spi}/store/SegmentDirectory.java | 34 ++-----
.../segment/spi}/store/SegmentDirectoryPaths.java | 10 +--
.../segment/spi}/memory/PinotDataBufferTest.java | 2 +-
.../spi}/partition/PartitionFunctionTest.java | 7 +-
.../server/api/resources/MmapDebugResource.java | 2 +-
.../api/resources/SegmentMetadataFetcher.java | 2 +-
.../pinot/server/api/resources/TablesResource.java | 2 +-
.../helix/HelixInstanceDataManagerConfig.java | 26 +++++-
.../server/starter/helix/HelixServerStarter.java | 2 +-
.../starter/helix/SegmentFetcherAndLoader.java | 2 +-
.../pinot/server/api/TablesResourceTest.java | 4 +-
.../config/instance/InstanceDataManagerConfig.java | 4 +
.../org/apache/pinot/tools/SegmentDumpTool.java | 2 +-
.../anonymizer/ArrayBasedGlobalDictionaries.java | 2 +-
.../pinot/tools/anonymizer/GlobalDictionaries.java | 2 +-
.../anonymizer/MapBasedGlobalDictionaries.java | 2 +-
.../anonymizer/PinotDataAndQueryAnonymizer.java | 4 +-
.../pinot/tools/perf/PerfBenchmarkRunner.java | 2 +-
.../apache/pinot/tools/scan/query/Aggregation.java | 2 +-
.../apache/pinot/tools/scan/query/Projection.java | 4 +-
.../tools/scan/query/SegmentQueryProcessor.java | 4 +-
.../apache/pinot/tools/scan/query/Selection.java | 2 +-
.../converter/DictionaryToRawIndexConverter.java | 2 +-
.../converter/PinotSegmentToAvroConverter.java | 2 +-
.../segment/converter/SegmentMergeCommand.java | 2 +-
255 files changed, 987 insertions(+), 667 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/pruner/PartitionZKMetadataPruner.java b/pinot-broker/src/main/java/org/apache/pinot/broker/pruner/PartitionZKMetadataPruner.java
index 3761c22..3975207 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/pruner/PartitionZKMetadataPruner.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/pruner/PartitionZKMetadataPruner.java
@@ -21,13 +21,13 @@ package org.apache.pinot.broker.pruner;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.request.FilterOperator;
import org.apache.pinot.common.utils.request.FilterQueryTree;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
/**
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/PartitionSegmentPruner.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/PartitionSegmentPruner.java
index 0bc18f8..f6544ef 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/PartitionSegmentPruner.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/PartitionSegmentPruner.java
@@ -31,7 +31,6 @@ import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.request.BrokerRequest;
import org.apache.pinot.common.request.Expression;
@@ -41,8 +40,9 @@ import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.common.utils.request.FilterQueryTree;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.pql.parsers.pql2.ast.FilterKind;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.CommonConstants.Segment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/pruner/SegmentZKMetadataPrunerTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/pruner/SegmentZKMetadataPrunerTest.java
index 29decea..0396086 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/pruner/SegmentZKMetadataPrunerTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/pruner/SegmentZKMetadataPrunerTest.java
@@ -21,12 +21,12 @@ package org.apache.pinot.broker.pruner;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.request.BrokerRequest;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SegmentPrunerTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SegmentPrunerTest.java
index 0f912e3..02cb866 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SegmentPrunerTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SegmentPrunerTest.java
@@ -34,7 +34,6 @@ import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
@@ -42,6 +41,7 @@ import org.apache.pinot.common.request.BrokerRequest;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.parsers.QueryCompiler;
import org.apache.pinot.pql.parsers.Pql2Compiler;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.RoutingConfig;
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentPartitionMetadata.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentPartitionMetadata.java
index b1a0770..22d4733 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentPartitionMetadata.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentPartitionMetadata.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.JsonUtils;
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentUtils.java
index 84c7a79..267f932 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/SegmentUtils.java
@@ -22,8 +22,8 @@ import com.google.common.base.Preconditions;
import java.util.Set;
import org.apache.helix.HelixManager;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
// Util functions related to segments.
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
index 918fd1d..6f835c6 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
@@ -26,11 +26,11 @@ import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.helix.ZNRecord;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
import org.apache.pinot.spi.utils.CommonConstants.Segment.SegmentType;
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
index c06f0d8..887770d 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadataTest.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.annotations.Test;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
index 3b6c3d3..9bab74a 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlers.java
@@ -47,8 +47,8 @@ import org.apache.pinot.controller.api.access.Authenticate;
import org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager;
import org.apache.pinot.controller.helix.core.realtime.segment.CommittingSegmentDescriptor;
import org.apache.pinot.controller.util.SegmentCompletionUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.filesystem.PinotFS;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineSegmentAssignment.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineSegmentAssignment.java
index fbc20a2..d89949d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineSegmentAssignment.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineSegmentAssignment.java
@@ -33,10 +33,10 @@ import org.apache.commons.configuration.Configuration;
import org.apache.helix.HelixManager;
import org.apache.pinot.common.assignment.InstancePartitions;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.tier.Tier;
import org.apache.pinot.controller.helix.core.rebalance.RebalanceConfigConstants;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ReplicaGroupStrategyConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index 2e1dcc3..d26a434 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -43,7 +43,6 @@ import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.pinot.common.assignment.InstancePartitions;
import org.apache.pinot.common.assignment.InstancePartitionsUtils;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metrics.ControllerMeter;
@@ -64,8 +63,9 @@ import org.apache.pinot.controller.helix.core.realtime.segment.CommittingSegment
import org.apache.pinot.controller.helix.core.realtime.segment.FlushThresholdUpdateManager;
import org.apache.pinot.controller.helix.core.realtime.segment.FlushThresholdUpdater;
import org.apache.pinot.controller.util.SegmentCompletionUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/CommittingSegmentDescriptor.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/CommittingSegmentDescriptor.java
index 36989be..94962a1 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/CommittingSegmentDescriptor.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/CommittingSegmentDescriptor.java
@@ -19,7 +19,7 @@
package org.apache.pinot.controller.helix.core.realtime.segment;
import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
/**
* Class to hold properties of the committing segment
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
index 324ea00..2aabb52 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/util/ZKMetadataUtils.java
@@ -20,14 +20,14 @@ package org.apache.pinot.controller.helix.core.util;
import java.util.HashMap;
import java.util.Map;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifier;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.utils.CommonConstants.Segment.SegmentType;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
index 252ba46..a73045a 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/realtime/provisioning/MemoryEstimator.java
@@ -47,11 +47,11 @@ import org.apache.pinot.segment.local.io.writer.impl.DirectMemoryManager;
import org.apache.pinot.segment.local.realtime.impl.RealtimeSegmentConfig;
import org.apache.pinot.segment.local.realtime.impl.RealtimeSegmentStatsHistory;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.DateTimeFormatSpec;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineReplicaGroupSegmentAssignmentTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineReplicaGroupSegmentAssignmentTest.java
index fa62039..11e0df0 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineReplicaGroupSegmentAssignmentTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/OfflineReplicaGroupSegmentAssignmentTest.java
@@ -31,10 +31,10 @@ import org.apache.helix.ZNRecord;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.pinot.common.assignment.InstancePartitions;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.controller.helix.core.rebalance.RebalanceConfigConstants;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ReplicaGroupStrategyConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index c3d58da..64c5cb8 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -47,8 +47,8 @@ import org.apache.pinot.controller.helix.core.assignment.segment.SegmentAssignme
import org.apache.pinot.controller.helix.core.realtime.segment.CommittingSegmentDescriptor;
import org.apache.pinot.controller.util.SegmentCompletionUtils;
import org.apache.pinot.core.realtime.impl.fakestream.FakeStreamConfigUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType;
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java b/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java
index 95e400a..3f18b9e 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/utils/SegmentMetadataMockUtils.java
@@ -24,10 +24,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
-import org.apache.pinot.segment.local.partition.MurmurPartitionFunction;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.partition.MurmurPartitionFunction;
import org.joda.time.Interval;
import org.mockito.Mockito;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index f7de67a..81ddc2b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -49,19 +49,19 @@ import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.segment.local.indexsegment.mutable.MutableSegmentImpl;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.local.realtime.converter.RealtimeSegmentConverter;
import org.apache.pinot.segment.local.realtime.impl.RealtimeSegmentConfig;
import org.apache.pinot.segment.local.recordtransformer.ComplexTypeTransformer;
import org.apache.pinot.segment.local.recordtransformer.CompositeTransformer;
import org.apache.pinot.segment.local.recordtransformer.RecordTransformer;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.local.upsert.PartitionUpsertMetadataManager;
import org.apache.pinot.segment.local.utils.IngestionUtils;
import org.apache.pinot.segment.spi.MutableSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.CompletionConfig;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/metadata/DefaultMetadataExtractor.java b/pinot-core/src/main/java/org/apache/pinot/core/metadata/DefaultMetadataExtractor.java
index 61835a9..18936d0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/metadata/DefaultMetadataExtractor.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/metadata/DefaultMetadataExtractor.java
@@ -20,8 +20,8 @@ package org.apache.pinot.core.metadata;
import java.io.File;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
/**
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
index a336811..21c594d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
@@ -29,16 +29,16 @@ import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoa
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.utils.CrcUtils;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
index 72482ed..401a376 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentConverter.java
@@ -32,9 +32,9 @@ import org.apache.pinot.core.minion.segment.RecordPartitioner;
import org.apache.pinot.core.minion.segment.ReducerRecordReader;
import org.apache.pinot.core.segment.processing.transformer.RecordTransformer;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
index b7f65d6..2b65508 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java
@@ -24,9 +24,9 @@ import java.io.IOException;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/rollup/MergeRollupSegmentConverter.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/rollup/MergeRollupSegmentConverter.java
index fc5aafa..518813e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/rollup/MergeRollupSegmentConverter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/rollup/MergeRollupSegmentConverter.java
@@ -27,7 +27,7 @@ import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.core.minion.SegmentConverter;
import org.apache.pinot.core.minion.segment.RecordAggregator;
import org.apache.pinot.core.segment.processing.transformer.RecordTransformer;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java
index 027e3b6..7fad934 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java
@@ -33,7 +33,7 @@ import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.core.util.GenericRowSerDeUtils;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/partitioner/TableConfigPartitioner.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/partitioner/TableConfigPartitioner.java
index 0f85134..589e557 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/partitioner/TableConfigPartitioner.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/partitioner/TableConfigPartitioner.java
@@ -18,8 +18,8 @@
*/
package org.apache.pinot.core.segment.processing.partitioner;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.partition.Partitioner;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.data.readers.GenericRow;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/GenericRowSerDeUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/GenericRowSerDeUtils.java
index a15fe28..2505c8c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/GenericRowSerDeUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/GenericRowSerDeUtils.java
@@ -20,7 +20,7 @@ package org.apache.pinot.core.util;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.StringUtils;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
index a54e159..2cf6702 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
@@ -21,21 +21,26 @@ package org.apache.pinot.core.minion;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.AfterClass;
@@ -155,8 +160,10 @@ public class SegmentPurgerTest {
}
// Check inverted index
- try (SegmentDirectory segmentDirectory = SegmentDirectory
- .createFromLocalFS(purgedIndexDir, purgedSegmentMetadata, ReadMode.mmap);
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(purgedIndexDir.toURI(), new PinotConfiguration(props));
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
assertTrue(reader.hasIndexFor(D1, ColumnIndexType.INVERTED_INDEX));
assertFalse(reader.hasIndexFor(D2, ColumnIndexType.INVERTED_INDEX));
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/pruner/ColumnValueSegmentPrunerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/pruner/ColumnValueSegmentPrunerTest.java
index 65b9ae0..d7ceced 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/pruner/ColumnValueSegmentPrunerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/pruner/ColumnValueSegmentPrunerTest.java
@@ -23,10 +23,10 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.testng.annotations.Test;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java
index 9982b42..fc7daef 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java
@@ -35,9 +35,9 @@ import org.apache.pinot.core.segment.processing.partitioner.PartitionerConfig;
import org.apache.pinot.core.segment.processing.partitioner.PartitionerFactory;
import org.apache.pinot.core.segment.processing.transformer.RecordTransformerConfig;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-core/src/test/java/org/apache/pinot/realtime/converter/RealtimeSegmentConverterTest.java b/pinot-core/src/test/java/org/apache/pinot/realtime/converter/RealtimeSegmentConverterTest.java
index 2616a34..aac84a7 100644
--- a/pinot-core/src/test/java/org/apache/pinot/realtime/converter/RealtimeSegmentConverterTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/realtime/converter/RealtimeSegmentConverterTest.java
@@ -30,9 +30,9 @@ import org.apache.pinot.segment.local.io.writer.impl.DirectMemoryManager;
import org.apache.pinot.segment.local.realtime.converter.RealtimeSegmentConverter;
import org.apache.pinot.segment.local.realtime.impl.RealtimeSegmentConfig;
import org.apache.pinot.segment.local.realtime.impl.RealtimeSegmentStatsHistory;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnProviderFactory;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ConvertToRawIndexMinionClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ConvertToRawIndexMinionClusterIntegrationTest.java
index 2b18127..1d96c71 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ConvertToRawIndexMinionClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ConvertToRawIndexMinionClusterIntegrationTest.java
@@ -32,9 +32,9 @@ import org.apache.pinot.controller.helix.core.minion.PinotHelixTaskResourceManag
import org.apache.pinot.controller.helix.core.minion.PinotTaskManager;
import org.apache.pinot.core.common.MinionConstants;
import org.apache.pinot.core.common.MinionConstants.ConvertToRawIndexTask;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableTaskConfig;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentPartitionLLCRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentPartitionLLCRealtimeClusterIntegrationTest.java
index b40e43e..1f1f1a1 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentPartitionLLCRealtimeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentPartitionLLCRealtimeClusterIntegrationTest.java
@@ -27,11 +27,11 @@ import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.SegmentPartitionMetadata;
import org.apache.pinot.common.utils.DataTable.MetadataKey;
import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.RoutingConfig;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitIntReader.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitIntReader.java
index 7785eb4..f0ff214 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitIntReader.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitIntReader.java
@@ -25,7 +25,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.reader.impl.FixedBitIntReader;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitSVForwardIndexReader.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitSVForwardIndexReader.java
index 7bec067..417eb63 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitSVForwardIndexReader.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFixedBitSVForwardIndexReader.java
@@ -26,7 +26,7 @@ import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReaderV2;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
index 3e01026..fc93f9d 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
@@ -22,15 +22,16 @@ import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.integration.tests.ClusterTest;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.DoubleDictionary;
import org.apache.pinot.segment.local.segment.index.readers.FloatDictionary;
import org.apache.pinot.segment.local.segment.index.readers.IntDictionary;
@@ -39,10 +40,14 @@ import org.apache.pinot.segment.local.segment.index.readers.StringDictionary;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitMVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.sorted.SortedIndexReaderImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.openjdk.jmh.annotations.Benchmark;
@@ -113,7 +118,10 @@ public class BenchmarkOfflineIndexReader {
File indexDir = new File(dataDir, TABLE_NAME);
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
- SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, segmentMetadata, ReadMode.mmap);
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(indexDir.toURI(), new PinotConfiguration(props));
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
// Forward index
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBitSet.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBitSet.java
index 364dfb0..eab7ac2 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBitSet.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBitSet.java
@@ -30,7 +30,7 @@ import org.apache.pinot.segment.local.io.util.PinotDataBitSetV2;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReaderV2;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java
index 7d0798f..5431954 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.nio.ByteOrder;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueryEngine.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueryEngine.java
index 6d51bab..e006203 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueryEngine.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueryEngine.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
import org.I0Itec.zkclient.ZkClient;
import org.apache.helix.ZNRecord;
import org.apache.helix.manager.zk.ZNRecordSerializer;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.tools.perf.PerfBenchmarkDriver;
import org.apache.pinot.tools.perf.PerfBenchmarkDriverConf;
import org.openjdk.jmh.annotations.Benchmark;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java b/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
index c87a89d..b6b17a5 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
@@ -37,9 +37,9 @@ import org.apache.pinot.core.operator.filter.TestFilterOperator;
import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.config.table.TableConfig;
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
index 5f2c4a3..b4e87fc 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
@@ -56,7 +56,7 @@ import org.apache.pinot.hadoop.utils.PinotHadoopJobPreparationHelper;
import org.apache.pinot.ingestion.common.ControllerRestApi;
import org.apache.pinot.ingestion.common.JobConfigConstants;
import org.apache.pinot.ingestion.jobs.SegmentPreprocessingJob;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java
index 1427a9a..0ba57db 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java
@@ -20,10 +20,10 @@ package org.apache.pinot.hadoop.job.partitioners;
import java.util.HashMap;
import java.util.Map;
-import org.apache.pinot.segment.local.partition.ByteArrayPartitionFunction;
-import org.apache.pinot.segment.local.partition.HashCodePartitionFunction;
-import org.apache.pinot.segment.local.partition.ModuloPartitionFunction;
-import org.apache.pinot.segment.local.partition.MurmurPartitionFunction;
+import org.apache.pinot.segment.spi.partition.ByteArrayPartitionFunction;
+import org.apache.pinot.segment.spi.partition.HashCodePartitionFunction;
+import org.apache.pinot.segment.spi.partition.ModuloPartitionFunction;
+import org.apache.pinot.segment.spi.partition.MurmurPartitionFunction;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtime_to_offline_segments/RealtimeToOfflineSegmentsTaskExecutorTest.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtime_to_offline_segments/RealtimeToOfflineSegmentsTaskExecutorTest.java
index 391549c..a4538a7 100644
--- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtime_to_offline_segments/RealtimeToOfflineSegmentsTaskExecutorTest.java
+++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtime_to_offline_segments/RealtimeToOfflineSegmentsTaskExecutorTest.java
@@ -35,10 +35,10 @@ import org.apache.pinot.core.minion.PinotTaskConfig;
import org.apache.pinot.minion.MinionContext;
import org.apache.pinot.plugin.minion.tasks.SegmentConversionResult;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
diff --git a/pinot-plugins/pinot-segment-writer/pinot-segment-writer-file-based/src/test/java/org/apache/pinot/plugin/segmentwriter/filebased/FileBasedSegmentWriterTest.java b/pinot-plugins/pinot-segment-writer/pinot-segment-writer-file-based/src/test/java/org/apache/pinot/plugin/segmentwriter/filebased/FileBasedSegmentWriterTest.java
index 5aa1b7c..7aaae70 100644
--- a/pinot-plugins/pinot-segment-writer/pinot-segment-writer-file-based/src/test/java/org/apache/pinot/plugin/segmentwriter/filebased/FileBasedSegmentWriterTest.java
+++ b/pinot-plugins/pinot-segment-writer/pinot-segment-writer-file-based/src/test/java/org/apache/pinot/plugin/segmentwriter/filebased/FileBasedSegmentWriterTest.java
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.ingestion.BatchIngestionConfig;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/EmptyIndexSegment.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/EmptyIndexSegment.java
index 6fa7526..c5d6b6e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/EmptyIndexSegment.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/EmptyIndexSegment.java
@@ -23,10 +23,10 @@ import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.segment.index.datasource.EmptyDataSource;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
index b8a3062..3dd6ef0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -26,21 +26,21 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.realtime.impl.ThreadSafeMutableRoaringBitmap;
import org.apache.pinot.segment.local.segment.index.datasource.ImmutableDataSource;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.ValidDocIndexReaderImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexContainer;
import org.apache.pinot.segment.local.upsert.PartitionUpsertMetadataManager;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
import org.apache.pinot.segment.spi.index.reader.ValidDocIndexReader;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 54d5841..1f2053d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -29,10 +29,6 @@ import org.apache.pinot.segment.local.segment.index.column.PhysicalColumnIndexCo
import org.apache.pinot.segment.local.segment.index.converter.SegmentFormatConverterFactory;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.SegmentPreProcessor;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnProvider;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnProviderFactory;
@@ -41,6 +37,12 @@ import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.ReadMode;
@@ -81,9 +83,10 @@ public class ImmutableSegmentLoader {
// NOTE: this step may modify the segment metadata
String segmentName = indexDir.getName();
SegmentVersion segmentVersionToLoad = indexLoadingConfig.getSegmentVersion();
+ SegmentMetadataImpl localSegmentMetadata = new SegmentMetadataImpl(indexDir);
if (segmentVersionToLoad != null && !SegmentDirectoryPaths.segmentDirectoryFor(indexDir, segmentVersionToLoad)
.isDirectory()) {
- SegmentVersion segmentVersionOnDisk = new SegmentMetadataImpl(indexDir).getSegmentVersion();
+ SegmentVersion segmentVersionOnDisk = localSegmentMetadata.getSegmentVersion();
if (segmentVersionOnDisk != segmentVersionToLoad) {
LOGGER.info("Segment: {} needs to be converted from version: {} to {}", segmentName, segmentVersionOnDisk,
segmentVersionToLoad);
@@ -96,17 +99,27 @@ public class ImmutableSegmentLoader {
}
}
- // Pre-process the segment
+ if (localSegmentMetadata.getTotalDocs() == 0) {
+ return new EmptyIndexSegment(localSegmentMetadata);
+ }
+
+ // Pre-process the segment on local using local SegmentDirectory
+ SegmentDirectory localSegmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(indexDir.toURI(), indexLoadingConfig.getTierConfigs());
+
// NOTE: this step may modify the segment metadata
- try (SegmentPreProcessor preProcessor = new SegmentPreProcessor(indexDir, indexLoadingConfig, schema)) {
+ try (
+ SegmentPreProcessor preProcessor = new SegmentPreProcessor(localSegmentDirectory, indexLoadingConfig, schema)) {
preProcessor.process();
}
- // Load the metadata again since converter and pre-processor may have changed it
- SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
- if (segmentMetadata.getTotalDocs() == 0) {
- return new EmptyIndexSegment(segmentMetadata);
- }
+ // Load the segment again for the configured tier backend. Default is 'local'.
+ SegmentDirectoryLoader segmentLoaderDirectory =
+ SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getTierBackend());
+ SegmentDirectory actualSegmentDirectory =
+ segmentLoaderDirectory.load(indexDir.toURI(), indexLoadingConfig.getTierConfigs());
+ SegmentDirectory.Reader segmentReader = actualSegmentDirectory.createReader();
+ SegmentMetadataImpl segmentMetadata = actualSegmentDirectory.getSegmentMetadata();
// Remove columns not in schema from the metadata
Map<String, ColumnMetadata> columnMetadataMap = segmentMetadata.getColumnMetadataMap();
@@ -121,12 +134,9 @@ public class ImmutableSegmentLoader {
}
}
- // Load the segment
- ReadMode readMode = indexLoadingConfig.getReadMode();
- SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, segmentMetadata, readMode);
- SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
Map<String, ColumnIndexContainer> indexContainerMap = new HashMap<>();
for (Map.Entry<String, ColumnMetadata> entry : columnMetadataMap.entrySet()) {
+ // FIXME: text-index only works with local SegmentDirectory
indexContainerMap.put(entry.getKey(),
new PhysicalColumnIndexContainer(segmentReader, entry.getValue(), indexLoadingConfig, indexDir));
}
@@ -144,17 +154,18 @@ public class ImmutableSegmentLoader {
}
}
+ // FIXME: star tree only works with local SegmentDirectory
// Load star-tree index if it exists
StarTreeIndexContainer starTreeIndexContainer = null;
if (segmentMetadata.getStarTreeV2MetadataList() != null) {
starTreeIndexContainer =
new StarTreeIndexContainer(SegmentDirectoryPaths.findSegmentDirectory(indexDir), segmentMetadata,
- indexContainerMap, readMode);
+ indexContainerMap, indexLoadingConfig.getReadMode());
}
ImmutableSegmentImpl segment =
- new ImmutableSegmentImpl(segmentDirectory, segmentMetadata, indexContainerMap, starTreeIndexContainer);
- LOGGER.info("Successfully loaded segment {} with readMode: {}", segmentName, readMode);
+ new ImmutableSegmentImpl(actualSegmentDirectory, segmentMetadata, indexContainerMap, starTreeIndexContainer);
+ LOGGER.info("Successfully loaded segment {} with config: {}", segmentName, indexLoadingConfig.getTierConfigs());
return segment;
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/IntermediateSegment.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/IntermediateSegment.java
index 8b5f663..ee0bea4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/IntermediateSegment.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/IntermediateSegment.java
@@ -32,15 +32,14 @@ import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
import org.apache.pinot.segment.local.io.writer.impl.MmapMemoryManager;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.local.realtime.impl.dictionary.MutableDictionaryFactory;
import org.apache.pinot.segment.local.realtime.impl.forward.FixedByteMVMutableForwardIndex;
import org.apache.pinot.segment.local.realtime.impl.forward.FixedByteSVMutableForwardIndex;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.column.IntermediateIndexContainer;
import org.apache.pinot.segment.local.segment.index.column.NumValuesInfo;
import org.apache.pinot.segment.spi.MutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.MutableDictionary;
@@ -48,6 +47,7 @@ import org.apache.pinot.segment.spi.index.reader.MutableForwardIndex;
import org.apache.pinot.segment.spi.index.reader.ValidDocIndexReader;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
index 982ebee..50dea95 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
@@ -34,6 +34,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.common.metrics.ServerMeter;
import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
@@ -51,10 +52,8 @@ import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLucene
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndexReader;
import org.apache.pinot.segment.local.realtime.impl.json.MutableJsonIndex;
import org.apache.pinot.segment.local.realtime.impl.nullvalue.MutableNullValueVector;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.datasource.ImmutableDataSource;
import org.apache.pinot.segment.local.segment.index.datasource.MutableDataSource;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.ValidDocIndexReaderImpl;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnProvider;
@@ -66,8 +65,10 @@ import org.apache.pinot.segment.local.utils.IdMap;
import org.apache.pinot.segment.local.utils.IngestionUtils;
import org.apache.pinot.segment.spi.MutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
import org.apache.pinot.segment.spi.index.reader.MutableDictionary;
@@ -170,8 +171,13 @@ public class MutableSegmentImpl implements MutableSegment {
_schema = config.getSchema();
_timeColumnName = config.getTimeColumnName();
_capacity = config.getCapacity();
- _segmentMetadata = new SegmentMetadataImpl(config.getRealtimeSegmentZKMetadata(), _schema) {
- @Override
+ final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = config.getRealtimeSegmentZKMetadata();
+ _segmentMetadata =
+ new SegmentMetadataImpl(realtimeSegmentZKMetadata.getTableName(), realtimeSegmentZKMetadata.getSegmentName(),
+ realtimeSegmentZKMetadata.getCreationTime(), realtimeSegmentZKMetadata.getStartTime(),
+ realtimeSegmentZKMetadata.getEndTime(), realtimeSegmentZKMetadata.getTimeUnit(),
+ realtimeSegmentZKMetadata.getTotalDocs(), realtimeSegmentZKMetadata.getCrc(), _schema) {
+ @Override
public int getTotalDocs() {
return _numDocsIndexed;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java
index e1ef922..997413d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.io.reader.impl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedByteSingleValueMultiColReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedByteSingleValueMultiColReader.java
index 5d5038a..0545a02 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedByteSingleValueMultiColReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedByteSingleValueMultiColReader.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.io.reader.impl;
import java.io.Closeable;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/PinotDataBufferMemoryManager.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/PinotDataBufferMemoryManager.java
index e71a79d..707d5d0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/PinotDataBufferMemoryManager.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/PinotDataBufferMemoryManager.java
@@ -19,7 +19,7 @@
package org.apache.pinot.segment.local.io.readerwriter;
import java.io.Closeable;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/RealtimeIndexOffHeapMemoryManager.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/RealtimeIndexOffHeapMemoryManager.java
index 8b759fb..bc03d5f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/RealtimeIndexOffHeapMemoryManager.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/RealtimeIndexOffHeapMemoryManager.java
@@ -27,7 +27,7 @@ import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.common.utils.HLCSegmentName;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.common.utils.SegmentName;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
index 56b9d29..aa48700 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.pinot.segment.local.io.reader.impl.FixedByteSingleValueMultiColReader;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriter.java
index 27e265b..76e1a15 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriter.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.io.util;
import com.google.common.base.Preconditions;
import java.io.Closeable;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public final class FixedBitIntReaderWriter implements Closeable {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriterV2.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriterV2.java
index 2eedb1e..07a67f6 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriterV2.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedBitIntReaderWriterV2.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.io.util;
import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public final class FixedBitIntReaderWriterV2 implements Closeable {
@@ -146,4 +146,4 @@ public final class FixedBitIntReaderWriterV2 implements Closeable {
_dataBitSet.close();
}
}
-}
\ No newline at end of file
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
index 1a8a816..f8ce398 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.io.util;
import java.util.Arrays;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public final class FixedByteValueReaderWriter implements ValueReader {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSet.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSet.java
index ca6803d..7cb840b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSet.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSet.java
@@ -19,7 +19,7 @@
package org.apache.pinot.segment.local.io.util;
import java.io.Closeable;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public final class PinotDataBitSet implements Closeable {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2.java
index c52fa38..208188b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.io.util;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public abstract class PinotDataBitSetV2 implements Closeable {
@@ -728,4 +728,4 @@ public abstract class PinotDataBitSetV2 implements Closeable {
public void close()
throws IOException {
}
-}
\ No newline at end of file
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
index 8307a0d..bc2e9c9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
@@ -19,7 +19,7 @@
package org.apache.pinot.segment.local.io.util;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueWriter.java
index e6c101f..c0c2194 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueWriter.java
@@ -25,7 +25,7 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.utils.CleanerUtil;
+import org.apache.pinot.segment.spi.memory.CleanerUtil;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/DirectMemoryManager.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/DirectMemoryManager.java
index 0a3f00a..06ffeff 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/DirectMemoryManager.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/DirectMemoryManager.java
@@ -22,7 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.pinot.common.metrics.PinotMetricUtils;
import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.segment.local.io.readerwriter.RealtimeIndexOffHeapMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
// Allocates memory using direct allocation
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitMVForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitMVForwardIndexWriter.java
index 9ba80d6..331438e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitMVForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitMVForwardIndexWriter.java
@@ -26,7 +26,7 @@ import java.nio.ByteOrder;
import org.apache.pinot.segment.local.io.util.FixedBitIntReaderWriter;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitSVForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitSVForwardIndexWriter.java
index 3ae68c9..b7c8c55 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitSVForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedBitSVForwardIndexWriter.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteOrder;
import org.apache.pinot.segment.local.io.util.FixedBitIntReaderWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteSingleValueMultiColWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteSingleValueMultiColWriter.java
index 989870c..32897b3 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteSingleValueMultiColWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteSingleValueMultiColWriter.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteOrder;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public class FixedByteSingleValueMultiColWriter implements Closeable {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManager.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManager.java
index 354c9d1..e495e77 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManager.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManager.java
@@ -29,7 +29,7 @@ import java.util.List;
import org.apache.pinot.common.metrics.PinotMetricUtils;
import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.segment.local.io.readerwriter.RealtimeIndexOffHeapMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MutableOffHeapByteArrayStore.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MutableOffHeapByteArrayStore.java
index a1f45fe..b7e5e63 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MutableOffHeapByteArrayStore.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/MutableOffHeapByteArrayStore.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java
new file mode 100644
index 0000000..7cf43c8
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.loader;
+
+import java.io.File;
+import java.net.URI;
+import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentLoader;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.ReadMode;
+
+
+/**
+ * Implementation of {@link SegmentDirectoryLoader} for local FS
+ */
+@SegmentLoader(name = "local")
+public class LocalSegmentDirectoryLoader implements SegmentDirectoryLoader {
+
+ public static final String READ_MODE_KEY = "readMode";
+
+ /**
+ * Creates and loads the {@link SegmentLocalFSDirectory} which is the local implementation of {@link SegmentDirectory}
+ * @param indexDir segment index directory
+ * @param segmentDirectoryConfig config containing values for instantiation of the SegmentDirectory
+ * @return instance of {@link SegmentLocalFSDirectory}
+ */
+ @Override
+ public SegmentDirectory load(URI indexDir, PinotConfiguration segmentDirectoryConfig)
+ throws Exception {
+ return new SegmentLocalFSDirectory(new File(indexDir), ReadMode.valueOf(segmentDirectoryConfig.getProperty(READ_MODE_KEY)));
+ }
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
index 99c8e87..89840bd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.MutableDictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/OffHeapMutableBytesStore.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/OffHeapMutableBytesStore.java
index 9421cee..f705e23 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/OffHeapMutableBytesStore.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/OffHeapMutableBytesStore.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicReferenceArray;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteMVMutableForwardIndex.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteMVMutableForwardIndex.java
index 6368af4..cd333df 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteMVMutableForwardIndex.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteMVMutableForwardIndex.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.pinot.segment.local.io.reader.impl.FixedByteSingleValueMultiColReader;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.MutableForwardIndex;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
index deb8124..ec62e4e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
@@ -25,8 +25,8 @@ import java.util.List;
import org.apache.pinot.segment.local.io.reader.impl.FixedByteSingleValueMultiColReader;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.MutableForwardIndex;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index a655a13..61b0017 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -34,7 +34,6 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.common.utils.FileUtils;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.TextIndexType;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.MultiValueUnsortedForwardIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueFixedByteRawIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueSortedForwardIndexCreator;
@@ -50,6 +49,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.inv.text.LuceneFSTInd
import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.utils.GeometrySerializer;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.creator.ColumnIndexCreationInfo;
import org.apache.pinot.segment.spi.creator.SegmentCreator;
@@ -61,6 +61,7 @@ import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
import org.apache.pinot.segment.spi.index.creator.SegmentIndexCreationInfo;
import org.apache.pinot.segment.spi.index.creator.TextIndexCreator;
+import org.apache.pinot.segment.spi.index.creator.TextIndexType;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.config.table.FieldConfig;
@@ -78,8 +79,8 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.MetadataKeys.Column.*;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.MetadataKeys.Segment.*;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.*;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Segment.*;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
index 0c2b58e..52ba98e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
@@ -32,7 +32,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.StringUtil;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
import org.apache.pinot.segment.local.io.util.VarLengthValueWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ByteArray;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentIndexCreationDriverImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentIndexCreationDriverImpl.java
index 94a6e39..54921eb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentIndexCreationDriverImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentIndexCreationDriverImpl.java
@@ -41,10 +41,10 @@ import org.apache.pinot.segment.local.segment.creator.RecordReaderSegmentCreatio
import org.apache.pinot.segment.local.segment.index.converter.SegmentFormatConverterFactory;
import org.apache.pinot.segment.local.segment.readers.IntermediateSegmentRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
import org.apache.pinot.segment.local.utils.CrcUtils;
import org.apache.pinot.segment.local.utils.IngestionUtils;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
import org.apache.pinot.segment.spi.creator.ColumnIndexCreationInfo;
import org.apache.pinot.segment.spi.creator.ColumnStatistics;
@@ -56,6 +56,7 @@ import org.apache.pinot.segment.spi.creator.SegmentPreIndexStatsContainer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.segment.spi.index.creator.SegmentIndexCreationInfo;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/bloom/OnHeapGuavaBloomFilterCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/bloom/OnHeapGuavaBloomFilterCreator.java
index fef54ac..c77817b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/bloom/OnHeapGuavaBloomFilterCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/bloom/OnHeapGuavaBloomFilterCreator.java
@@ -25,8 +25,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.readers.bloom.GuavaBloomFilterReaderUtils;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.BloomFilterCreator;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueUnsortedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueUnsortedForwardIndexCreator.java
index 7e4e073..62b6f9a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueUnsortedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueUnsortedForwardIndexCreator.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitMVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
index ebf7eaf..489be39 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueSortedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueSortedForwardIndexCreator.java
index ee8218c..8543bda 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueSortedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueSortedForwardIndexCreator.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueUnsortedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueUnsortedForwardIndexCreator.java
index 6c1715a..0ae68ff 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueUnsortedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueUnsortedForwardIndexCreator.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
index c5fc999..072061c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/BitmapInvertedIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/BitmapInvertedIndexWriter.java
index 452f022..a00256d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/BitmapInvertedIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/BitmapInvertedIndexWriter.java
@@ -25,7 +25,7 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
-import org.apache.pinot.segment.local.utils.CleanerUtil;
+import org.apache.pinot.segment.spi.memory.CleanerUtil;
import org.roaringbitmap.RoaringBitmap;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OffHeapBitmapInvertedIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OffHeapBitmapInvertedIndexCreator.java
index f0e810d..f75574d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OffHeapBitmapInvertedIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OffHeapBitmapInvertedIndexCreator.java
@@ -22,9 +22,9 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCreator;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapWriter;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OnHeapBitmapInvertedIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OnHeapBitmapInvertedIndexCreator.java
index a069900..e285a17 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OnHeapBitmapInvertedIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/OnHeapBitmapInvertedIndexCreator.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.segment.creator.impl.inv;
import java.io.File;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCreator;
import org.roaringbitmap.Container;
import org.roaringbitmap.RoaringBitmap;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
index d74b4a8..3f120d8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
@@ -33,16 +33,16 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCreator;
import org.apache.pinot.segment.spi.index.creator.RawValueBasedInvertedIndexCreator;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.Pair;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION;
import static org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
index 09ffaec..1bb2562 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
@@ -30,8 +30,8 @@ import java.nio.channels.FileChannel;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.utils.H3Utils;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.GeoSpatialIndexCreator;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.locationtech.jts.geom.Coordinate;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
index f8123e0..b4ee786 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
@@ -31,8 +31,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.locationtech.jts.geom.Geometry;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapWriter;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/BaseJsonIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/BaseJsonIndexCreator.java
index 3cfc4b5..5fc35b4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/BaseJsonIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/BaseJsonIndexCreator.java
@@ -31,9 +31,9 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.utils.CleanerUtil;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
+import org.apache.pinot.segment.spi.memory.CleanerUtil;
import org.apache.pinot.spi.utils.JsonUtils;
import org.roaringbitmap.Container;
import org.roaringbitmap.RoaringBitmap;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java
index 56e9924..89d5887 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java
@@ -33,8 +33,8 @@ import java.util.Map;
import java.util.PriorityQueue;
import org.apache.pinot.segment.local.io.util.VarLengthValueWriter;
import org.apache.pinot.segment.local.segment.creator.impl.inv.BitmapInvertedIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.StringUtils;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapWriter;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
index 81f0429..5e78289 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
@@ -24,8 +24,8 @@ import java.io.IOException;
import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.fst.FST;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.utils.fst.FSTBuilder;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.TextIndexCreator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
index 68cf437..4ccd83e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
@@ -24,7 +24,7 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
index 93f6283..7d7b085 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
@@ -21,10 +21,10 @@ package org.apache.pinot.segment.local.segment.creator.impl.stats;
import java.util.HashSet;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
import org.apache.pinot.segment.spi.creator.ColumnStatistics;
import org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index c6d6595..aa4e9b5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -33,6 +33,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndexReader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCreator;
import org.apache.pinot.segment.spi.index.creator.TextIndexCreator;
@@ -47,7 +48,6 @@ public class LuceneTextIndexCreator implements TextIndexCreator {
private static final int LUCENE_INDEX_MAX_BUFFER_SIZE_MB = 500;
public static final String LUCENE_INDEX_DOC_ID_COLUMN_NAME = "DocID";
- public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
private final String _textColumn;
private final Directory _indexDirectory;
@@ -157,7 +157,7 @@ public class LuceneTextIndexCreator implements TextIndexCreator {
}
private File getV1TextIndexFile(File indexDir) {
- String luceneIndexDirectory = _textColumn + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String luceneIndexDirectory = _textColumn + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
return new File(indexDir, luceneIndexDirectory);
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/BaseVirtualColumnProvider.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/BaseVirtualColumnProvider.java
index e7837ab..4c54e57 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/BaseVirtualColumnProvider.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/BaseVirtualColumnProvider.java
@@ -18,11 +18,11 @@
*/
package org.apache.pinot.segment.local.segment.index.column;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnIndexContainer;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnProvider;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProvider.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProvider.java
index 3488f1c..0405547 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProvider.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProvider.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.segment.index.column;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueBytesDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueDoubleDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueFloatDictionary;
@@ -29,6 +28,7 @@ import org.apache.pinot.segment.local.segment.index.readers.constant.ConstantMVF
import org.apache.pinot.segment.local.segment.index.readers.constant.ConstantMVInvertedIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.constant.ConstantSortedIndexReader;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
index d5ad483..43ab10c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.BytesDictionary;
@@ -49,10 +48,8 @@ import org.apache.pinot.segment.local.segment.index.readers.geospatial.Immutable
import org.apache.pinot.segment.local.segment.index.readers.json.ImmutableJsonIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.sorted.SortedIndexReaderImpl;
import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
@@ -61,6 +58,9 @@ import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.segment.spi.index.reader.SortedIndexReader;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index 497bbec..129d968 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -26,22 +26,25 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.attribute.PosixFilePermission;
import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,7 +107,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
if (file.isFile() && file.exists()) {
FileUtils.deleteQuietly(file);
}
- if (file.isDirectory() && file.getName().endsWith(LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION)) {
+ if (file.isDirectory() && file.getName().endsWith(V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION)) {
FileUtils.deleteDirectory(file);
}
}
@@ -134,9 +137,13 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
private void copyIndexData(File v2Directory, SegmentMetadataImpl v2Metadata, File v3Directory)
throws Exception {
- SegmentMetadataImpl v3Metadata = new SegmentMetadataImpl(v3Directory);
- try (SegmentDirectory v2Segment = SegmentDirectory.createFromLocalFS(v2Directory, v2Metadata, ReadMode.mmap);
- SegmentDirectory v3Segment = SegmentDirectory.createFromLocalFS(v3Directory, v3Metadata, ReadMode.mmap)) {
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ PinotConfiguration configuration = new PinotConfiguration(props);
+ try (SegmentDirectory v2Segment = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(v2Directory.toURI(), configuration);
+ SegmentDirectory v3Segment = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(v3Directory.toURI(), configuration)) {
// for each dictionary and each fwdIndex, copy that to newDirectory buffer
Set<String> allColumns = v2Metadata.getAllColumns();
@@ -212,7 +219,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
private void copyLuceneTextIndexIfExists(File segmentDirectory, File v3Dir)
throws IOException {
// TODO: see if this can be done by reusing some existing methods
- String suffix = LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String suffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
File[] textIndexFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -233,7 +240,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
// to moving the lucene text index files, we need to move the
// docID mapping/cache file created by us in v1/v2 during an earlier
// load of the segment.
- String docIDFileSuffix = LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ String docIDFileSuffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
File[] textIndexDocIdMappingFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/EmptyDataSource.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/EmptyDataSource.java
index d53f345..52adf88 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/EmptyDataSource.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/EmptyDataSource.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.datasource;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java
index 624dd19..633ff20 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java
@@ -20,9 +20,9 @@ package org.apache.pinot.segment.local.segment.index.datasource;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
index 7bfaa68..3604ffa 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.index.column.PhysicalColumnIndexContainer;
import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
@@ -36,6 +38,7 @@ import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.ReadMode;
@@ -45,6 +48,7 @@ import org.apache.pinot.spi.utils.ReadMode;
*/
public class IndexLoadingConfig {
private static final int DEFAULT_REALTIME_AVG_MULTI_VALUE_COUNT = 2;
+ public static final String DEFAULT_TIER_BACKEND = "local";
private ReadMode _readMode = ReadMode.DEFAULT_MODE;
private List<String> _sortedColumns = Collections.emptyList();
@@ -76,6 +80,8 @@ public class IndexLoadingConfig {
private Map<String, Map<String, String>> _columnProperties = new HashMap<>();
private TableConfig _tableConfig;
+ private String _tierBackend;
+ private PinotConfiguration _tierConfigs;
public IndexLoadingConfig(InstanceDataManagerConfig instanceDataManagerConfig, TableConfig tableConfig) {
extractFromInstanceConfig(instanceDataManagerConfig);
@@ -237,6 +243,8 @@ public class IndexLoadingConfig {
}
_enableSplitCommitEndWithMetadata = instanceDataManagerConfig.isEnableSplitCommitEndWithMetadata();
_segmentStoreURI = instanceDataManagerConfig.getConfig().getProperty(CommonConstants.Server.CONFIG_OF_SEGMENT_STORE_URI);
+ _tierBackend = instanceDataManagerConfig.getTierBackend();
+ _tierConfigs = instanceDataManagerConfig.getTierConfigs();
}
/**
@@ -270,10 +278,8 @@ public class IndexLoadingConfig {
/**
* Used in two places:
- * (1) In {@link PhysicalColumnIndexContainer}
- * to create the index loading info for immutable segments
- * (2) In {@link org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager}
- * to create the {@link org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig}.
+ * (1) In {@link PhysicalColumnIndexContainer} to create the index loading info for immutable segments
+ * (2) In LLRealtimeSegmentDataManager to create the RealtimeSegmentConfig.
* RealtimeSegmentConfig is used to specify the text index column info for newly
* to-be-created Mutable Segments
* @return a set containing names of text index columns
@@ -442,4 +448,17 @@ public class IndexLoadingConfig {
public void setTableConfig(TableConfig tableConfig) {
_tableConfig = tableConfig;
}
+
+ public String getTierBackend() {
+ return StringUtils.isNotBlank(_tierBackend) ? _tierBackend : DEFAULT_TIER_BACKEND;
+ }
+
+ public PinotConfiguration getTierConfigs() {
+ if (_tierConfigs == null) {
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, _readMode);
+ return new PinotConfiguration(props);
+ }
+ return _tierConfigs;
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
index 3b63bb7..8e47f9f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/LoaderUtils.java
@@ -27,17 +27,17 @@ import javax.annotation.Nonnull;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.index.column.PhysicalColumnIndexContainer;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitMVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReaderV2;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedByteChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.sorted.SortedIndexReaderImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.CommonConstants;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
index 917bd8b..8b1347f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.loader.bloomfilter.BloomFilterHandler;
import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGenerator;
import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode;
@@ -35,14 +34,14 @@ import org.apache.pinot.segment.local.segment.index.loader.invertedindex.JsonInd
import org.apache.pinot.segment.local.segment.index.loader.invertedindex.LuceneFSTIndexHandler;
import org.apache.pinot.segment.local.segment.index.loader.invertedindex.RangeIndexHandler;
import org.apache.pinot.segment.local.segment.index.loader.invertedindex.TextIndexHandler;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.local.startree.StarTreeBuilderUtils;
import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,16 +64,13 @@ public class SegmentPreProcessor implements AutoCloseable {
private final SegmentDirectory _segmentDirectory;
private SegmentMetadataImpl _segmentMetadata;
- public SegmentPreProcessor(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema)
+ public SegmentPreProcessor(SegmentDirectory segmentDirectory, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema)
throws Exception {
- _indexDir = indexDir;
+ _segmentDirectory = segmentDirectory;
+ _indexDir = new File(segmentDirectory.getIndexDir());
_indexLoadingConfig = indexLoadingConfig;
_schema = schema;
- _segmentMetadata = new SegmentMetadataImpl(indexDir);
-
- // Always use mmap to load the segment because it is safest and performs well without impact from -Xmx params.
- // This is not the final load of the segment.
- _segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, _segmentMetadata, ReadMode.mmap);
+ _segmentMetadata = segmentDirectory.getSegmentMetadata();
}
public void process()
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
index 104caef..c2c8927 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
@@ -24,12 +24,9 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.bloom.OnHeapGuavaBloomFilterCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.segment.local.segment.index.readers.BytesDictionary;
import org.apache.pinot.segment.local.segment.index.readers.DoubleDictionary;
@@ -37,12 +34,15 @@ import org.apache.pinot.segment.local.segment.index.readers.FloatDictionary;
import org.apache.pinot.segment.local.segment.index.readers.IntDictionary;
import org.apache.pinot.segment.local.segment.index.readers.LongDictionary;
import org.apache.pinot.segment.local.segment.index.readers.StringDictionary;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.creator.BloomFilterCreator;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
index 8549893..7c7e442 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
@@ -24,17 +24,17 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.BytesDictionary;
import org.apache.pinot.segment.local.segment.index.readers.DoubleDictionary;
import org.apache.pinot.segment.local.segment.index.readers.FloatDictionary;
import org.apache.pinot.segment.local.segment.index.readers.IntDictionary;
import org.apache.pinot.segment.local.segment.index.readers.LongDictionary;
import org.apache.pinot.segment.local.segment.index.readers.StringDictionary;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.Schema;
import static org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 71c549c..351b4f1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -36,10 +36,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.StringUtil;
import org.apache.pinot.segment.local.function.FunctionEvaluator;
import org.apache.pinot.segment.local.function.FunctionEvaluatorFactory;
-import org.apache.pinot.segment.local.segment.creator.TextIndexType;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.MultiValueUnsortedForwardIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueSortedForwardIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueUnsortedForwardIndexCreator;
@@ -51,15 +49,17 @@ import org.apache.pinot.segment.local.segment.creator.impl.stats.LongColumnPreIn
import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.ColumnIndexCreationInfo;
import org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
+import org.apache.pinot.segment.spi.index.creator.TextIndexType;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.ingestion.TransformConfig;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnHandlerFactory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnHandlerFactory.java
index 2363368..a446ace 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnHandlerFactory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnHandlerFactory.java
@@ -20,9 +20,9 @@ package org.apache.pinot.segment.local.segment.index.loader.defaultcolumn;
import java.io.File;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.Schema;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V1DefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V1DefaultColumnHandler.java
index 2a79208..10030d9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V1DefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V1DefaultColumnHandler.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.loader.defaultcolumn;
import java.io.File;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V3DefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V3DefaultColumnHandler.java
index c983355..bb98f59 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V3DefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/V3DefaultColumnHandler.java
@@ -20,14 +20,14 @@ package org.apache.pinot.segment.local.segment.index.loader.defaultcolumn;
import com.google.common.base.Preconditions;
import java.io.File;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
import org.apache.pinot.segment.local.segment.index.loader.V3RemoveIndexException;
import org.apache.pinot.segment.local.segment.index.loader.V3UpdateIndexException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
index 6dd7721..bb74b24 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
@@ -26,20 +26,20 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.OffHeapH3IndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.local.utils.GeometrySerializer;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
index cc18c8d..625d4ea 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
@@ -23,17 +23,17 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.OffHeapBitmapInvertedIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
index b90732f..3773989 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
@@ -25,18 +25,18 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OffHeapJsonIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java
index 5858ece..8add430 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java
@@ -27,17 +27,17 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.inv.text.LuceneFSTIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
import org.apache.pinot.segment.local.segment.index.loader.SegmentPreProcessor;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
index c77db3f..60331a2 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
@@ -23,17 +23,17 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.RangeIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
index 5b2eeeb..3a06f17 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
@@ -40,27 +40,27 @@ import java.io.File;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.pinot.segment.local.segment.creator.TextIndexType;
import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
import org.apache.pinot.segment.local.segment.index.loader.SegmentPreProcessor;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.forward.BaseChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.creator.TextIndexType;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.MetadataKeys.Column.TEXT_INDEX_TYPE;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.MetadataKeys.Column.getKeyFor;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.TEXT_INDEX_TYPE;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.getKeyFor;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
index db8b40b..ee91995 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
@@ -26,8 +26,8 @@ import org.apache.pinot.common.utils.StringUtil;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
import org.apache.pinot.segment.local.io.util.ValueReader;
import org.apache.pinot.segment.local.io.util.VarLengthValueReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.ByteArray;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitmapInvertedIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitmapInvertedIndexReader.java
index 7703b04..7226054 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitmapInvertedIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitmapInvertedIndexReader.java
@@ -21,8 +21,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import java.lang.ref.SoftReference;
import java.nio.ByteOrder;
import org.apache.pinot.segment.local.segment.creator.impl.inv.BitmapInvertedIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
index 509eae0..b54256a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/DoubleDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/DoubleDictionary.java
index 85914ea..029752b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/DoubleDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/DoubleDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/FloatDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/FloatDictionary.java
index 625446f..65a33df 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/FloatDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/FloatDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/IntDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/IntDictionary.java
index 935d5b5..8d11ccd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/IntDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/IntDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LongDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LongDictionary.java
index 3d39996..df489be 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LongDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LongDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
index 0797554..8a230b9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
@@ -23,10 +23,10 @@ import java.util.List;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.OffHeapFSTStore;
import org.apache.lucene.util.fst.PositiveIntOutputs;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.local.utils.fst.PinotBufferIndexInput;
import org.apache.pinot.segment.local.utils.fst.RegexpMatcher;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImpl.java
index b388956..7a60cd8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImpl.java
@@ -18,8 +18,8 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDictionary.java
index 05d11c6..0390397 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDoubleDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDoubleDictionary.java
index 992b30d..e0a0876 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDoubleDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapDoubleDictionary.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import it.unimi.dsi.fastutil.doubles.Double2IntOpenHashMap;
import java.util.Arrays;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapFloatDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapFloatDictionary.java
index 4b60198..f90d0d1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapFloatDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapFloatDictionary.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import it.unimi.dsi.fastutil.floats.Float2IntOpenHashMap;
import java.util.Arrays;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapIntDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapIntDictionary.java
index 71df346..85335bd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapIntDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapIntDictionary.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import java.util.Arrays;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapLongDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapLongDictionary.java
index 714d7b8..43cc577 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapLongDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapLongDictionary.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import java.util.Arrays;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapStringDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapStringDictionary.java
index 3f53c0b..58cea67 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapStringDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/OnHeapStringDictionary.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import java.util.Arrays;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BytesUtils;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RangeIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RangeIndexReader.java
index bd71365..feaf02c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RangeIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RangeIndexReader.java
@@ -22,8 +22,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
index a431153..91ee274 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BytesUtils;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BaseGuavaBloomFilterReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BaseGuavaBloomFilterReader.java
index c4861ad..0658056 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BaseGuavaBloomFilterReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BaseGuavaBloomFilterReader.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers.bloom;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BloomFilterReaderFactory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BloomFilterReaderFactory.java
index 38445e3..915ace5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BloomFilterReaderFactory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/BloomFilterReaderFactory.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.index.readers.bloom;
import com.google.common.base.Preconditions;
import org.apache.pinot.segment.local.segment.creator.impl.bloom.OnHeapGuavaBloomFilterCreator;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
public class BloomFilterReaderFactory {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OffHeapGuavaBloomFilterReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OffHeapGuavaBloomFilterReader.java
index e8903dc..16457e1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OffHeapGuavaBloomFilterReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OffHeapGuavaBloomFilterReader.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers.bloom;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OnHeapGuavaBloomFilterReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OnHeapGuavaBloomFilterReader.java
index d4d8a80..9c687a0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OnHeapGuavaBloomFilterReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/bloom/OnHeapGuavaBloomFilterReader.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers.bloom;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkSVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkSVForwardIndexReader.java
index c7fe2f5..ba62706 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkSVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkSVForwardIndexReader.java
@@ -23,12 +23,12 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.pinot.segment.local.io.compression.ChunkCompressorFactory;
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.utils.CleanerUtil;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.compression.ChunkDecompressor;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.memory.CleanerUtil;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
index 9c3c869..784bcf2 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
@@ -21,9 +21,9 @@ package org.apache.pinot.segment.local.segment.index.readers.forward;
import org.apache.pinot.segment.local.io.util.FixedBitIntReaderWriter;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
import org.apache.pinot.segment.local.io.util.PinotDataBitSet;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReader.java
index 2502e50..a0122bc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReader.java
@@ -19,9 +19,9 @@
package org.apache.pinot.segment.local.segment.index.readers.forward;
import org.apache.pinot.segment.local.io.util.FixedBitIntReaderWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2.java
index f56fb47..83691f5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2.java
@@ -19,9 +19,9 @@
package org.apache.pinot.segment.local.segment.index.readers.forward;
import org.apache.pinot.segment.local.io.reader.impl.FixedBitIntReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedByteChunkSVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedByteChunkSVForwardIndexReader.java
index adb1065..94d8a9a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedByteChunkSVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedByteChunkSVForwardIndexReader.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.segment.index.readers.forward;
import java.nio.ByteBuffer;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
index ee5c35b..e4b2785 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
@@ -22,7 +22,7 @@ import java.nio.ByteBuffer;
import javax.annotation.Nullable;
import org.apache.pinot.common.utils.StringUtil;
import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/geospatial/ImmutableH3IndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/geospatial/ImmutableH3IndexReader.java
index 2619e7c..c046621 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/geospatial/ImmutableH3IndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/geospatial/ImmutableH3IndexReader.java
@@ -24,9 +24,9 @@ import java.nio.ByteOrder;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.BaseH3IndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.LongDictionary;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
index aa6532b..bf50355 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/json/ImmutableJsonIndexReader.java
@@ -32,8 +32,8 @@ import org.apache.pinot.common.request.context.predicate.Predicate;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.BaseJsonIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.StringDictionary;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.exception.BadQueryRequestException;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
index 4349b02..e508b0d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
@@ -21,9 +21,9 @@ package org.apache.pinot.segment.local.segment.index.readers.sorted;
import com.google.common.base.Preconditions;
import java.io.IOException;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.segment.spi.index.reader.SortedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.Pairs;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
index f221129..ae49e68 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
@@ -37,9 +37,10 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.segment.index.column.PhysicalColumnIndexContainer;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
@@ -62,8 +63,6 @@ public class LuceneTextIndexReader implements TextIndexReader {
private final StandardAnalyzer _standardAnalyzer;
private boolean _useANDForMultiTermQueries = false;
- public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
-
/**
* As part of loading the segment in ImmutableSegmentLoader,
* we load the text index (per column if it exists) and store
@@ -180,7 +179,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
throws Exception {
int length = Integer.BYTES * numDocs;
File docIdMappingFile = new File(SegmentDirectoryPaths.findSegmentDirectory(segmentIndexDir),
- column + LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
// The mapping is local to a segment. It is created on the server during segment load.
// Unless we are running Pinot on Solaris/SPARC, the underlying architecture is
// LITTLE_ENDIAN (Linux/x86). So use that as byte order.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectory.java
index 067c0a8..d38ebe0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectory.java
@@ -25,10 +25,11 @@ import java.io.IOException;
import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.Map;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.spi.utils.ReadMode;
@@ -130,7 +131,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
filename = metadata.getNullValueVectorFileName(column);
break;
case TEXT_INDEX:
- filename = column + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ filename = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
break;
case FST_INDEX:
filename = column + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexEntry.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexEntry.java
index 2c7197e..6016866 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexEntry.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexEntry.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.store;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexKey.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexKey.java
index 3d83f30..7de4e93 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexKey.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/IndexKey.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.store;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
index 9c25454..3a21ae2 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectory.java
@@ -18,16 +18,22 @@
*/
package org.apache.pinot.segment.local.segment.store;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,43 +44,42 @@ import static org.apache.pinot.segment.spi.creator.SegmentVersion.v3;
import static org.apache.pinot.segment.spi.creator.SegmentVersion.valueOf;
-class SegmentLocalFSDirectory extends SegmentDirectory {
- private static Logger LOGGER = LoggerFactory.getLogger(SegmentLocalFSDirectory.class);
+public class SegmentLocalFSDirectory extends SegmentDirectory {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SegmentLocalFSDirectory.class);
// matches most systems
private static final int PAGE_SIZE_BYTES = 4096;
// Prefetch limit...arbitrary but related to common server memory and data size profiles
private static final long MAX_MMAP_PREFETCH_PAGES = 100 * 1024 * 1024 * 1024L / PAGE_SIZE_BYTES;
private static final double PREFETCH_SLOWDOWN_PCT = 0.67;
- private static AtomicLong prefetchedPages = new AtomicLong(0);
+ private static final AtomicLong prefetchedPages = new AtomicLong(0);
- private final File segmentDirectory;
+ private final File _indexDir;
+ private final File _segmentDirectory;
SegmentLock segmentLock;
private SegmentMetadataImpl segmentMetadata;
- private ReadMode readMode;
+ private final ReadMode _readMode;
private ColumnIndexDirectory columnIndexDirectory;
- SegmentLocalFSDirectory(String directoryPath, SegmentMetadataImpl metadata, ReadMode readMode) {
- this(new File(directoryPath), metadata, readMode);
- }
-
- SegmentLocalFSDirectory(File directory, ReadMode readMode)
- throws IOException, ConfigurationException {
- this(directory, loadSegmentMetadata(directory), readMode);
+ public SegmentLocalFSDirectory(File directory, ReadMode readMode)
+ throws IOException {
+ this(directory, new SegmentMetadataImpl(directory), readMode);
}
- SegmentLocalFSDirectory(File directoryFile, SegmentMetadataImpl metadata, ReadMode readMode) {
+ @VisibleForTesting
+ public SegmentLocalFSDirectory(File directoryFile, SegmentMetadataImpl metadata, ReadMode readMode) {
Preconditions.checkNotNull(directoryFile);
Preconditions.checkNotNull(metadata);
- segmentDirectory = getSegmentPath(directoryFile, metadata.getSegmentVersion());
- Preconditions.checkState(segmentDirectory.exists(), "Segment directory: " + directoryFile + " must exist");
+ _indexDir = directoryFile;
+ _segmentDirectory = getSegmentPath(directoryFile, metadata.getSegmentVersion());
+ Preconditions.checkState(_segmentDirectory.exists(), "Segment directory: " + directoryFile + " must exist");
segmentLock = new SegmentLock();
- this.segmentMetadata = metadata;
- this.readMode = readMode;
+ segmentMetadata = metadata;
+ _readMode = readMode;
try {
load();
} catch (IOException | ConfigurationException e) {
@@ -84,10 +89,20 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
}
@Override
+ public URI getIndexDir() {
+ return _indexDir.toURI();
+ }
+
+ @Override
+ public SegmentMetadataImpl getSegmentMetadata() {
+ return segmentMetadata;
+ }
+
+ @Override
public void reloadMetadata()
throws Exception {
- this.segmentMetadata = loadSegmentMetadata(segmentDirectory);
- columnIndexDirectory.metadata = this.segmentMetadata;
+ segmentMetadata = new SegmentMetadataImpl(_indexDir);
+ columnIndexDirectory.setMetadata(segmentMetadata);
}
private File getSegmentPath(File segmentDirectory, SegmentVersion segmentVersion) {
@@ -109,14 +124,9 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
throw new IllegalArgumentException("Unknown segment version: " + segmentVersion);
}
- public static SegmentMetadataImpl loadSegmentMetadata(File segmentDirectory)
- throws IOException, ConfigurationException {
- return new SegmentMetadataImpl(segmentDirectory);
- }
-
@Override
public Path getPath() {
- return segmentDirectory.toPath();
+ return _segmentDirectory.toPath();
}
@Override
@@ -128,23 +138,23 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
// or default columns will not exist.
// check that v3 subdirectory exists since the format may not have been converted
- if (segmentDirectory.exists()) {
+ if (_segmentDirectory.exists()) {
try {
- return FileUtils.sizeOfDirectory(segmentDirectory.toPath().toFile());
+ return FileUtils.sizeOfDirectory(_segmentDirectory.toPath().toFile());
} catch (IllegalArgumentException e) {
- LOGGER.error("Failed to read disk size for directory: ", segmentDirectory.getAbsolutePath());
+ LOGGER.error("Failed to read disk size for directory: ", _segmentDirectory.getAbsolutePath());
return -1;
}
} else {
- if (!SegmentDirectoryPaths.isV3Directory(segmentDirectory)) {
+ if (!SegmentDirectoryPaths.isV3Directory(_segmentDirectory)) {
LOGGER
- .error("Segment directory: {} not found on disk and is not v3 format", segmentDirectory.getAbsolutePath());
+ .error("Segment directory: {} not found on disk and is not v3 format", _segmentDirectory.getAbsolutePath());
return -1;
}
- File[] files = segmentDirectory.getParentFile().listFiles();
+ File[] files = _segmentDirectory.getParentFile().listFiles();
if (files == null) {
- LOGGER.warn("Empty list of files for path: {}, segmentDirectory: {}", segmentDirectory.getParentFile(),
- segmentDirectory);
+ LOGGER.warn("Empty list of files for path: {}, segmentDirectory: {}", _segmentDirectory.getParentFile(),
+ _segmentDirectory);
return -1;
}
@@ -181,7 +191,7 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
@Override
public String toString() {
- return segmentDirectory.toString();
+ return _segmentDirectory.toString();
}
protected void load()
@@ -202,11 +212,11 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
switch (segmentVersion) {
case v1:
case v2:
- columnIndexDirectory = new FilePerIndexDirectory(segmentDirectory, segmentMetadata, readMode);
+ columnIndexDirectory = new FilePerIndexDirectory(_segmentDirectory, segmentMetadata, _readMode);
break;
case v3:
try {
- columnIndexDirectory = new SingleFileIndexDirectory(segmentDirectory, segmentMetadata, readMode);
+ columnIndexDirectory = new SingleFileIndexDirectory(_segmentDirectory, segmentMetadata, _readMode);
} catch (ConfigurationException e) {
LOGGER.error("Failed to create columnar index directory", e);
throw new RuntimeException(e);
@@ -233,7 +243,7 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
buffer = columnIndexDirectory.getBuffer(column, type);
- if (readMode == ReadMode.mmap) {
+ if (_readMode == ReadMode.mmap) {
prefetchMmapData(buffer);
}
return buffer;
@@ -306,7 +316,7 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
@Override
public String toString() {
- return segmentDirectory.toString();
+ return _segmentDirectory.toString();
}
}
@@ -346,7 +356,7 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
@Override
public String toString() {
- return segmentDirectory.toString();
+ return _segmentDirectory.toString();
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
index 7a863d1..0abe69b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectory.java
@@ -34,9 +34,11 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
@@ -114,7 +116,7 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory {
}
private boolean hasTextIndex(String column) {
- String suffix = LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String suffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
File[] textIndexFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/DocIdVirtualColumnProvider.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/DocIdVirtualColumnProvider.java
index 2cc90a4..1d1cb5c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/DocIdVirtualColumnProvider.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/DocIdVirtualColumnProvider.java
@@ -20,8 +20,8 @@ package org.apache.pinot.segment.local.segment.virtualcolumn;
import java.io.IOException;
import org.apache.pinot.segment.local.segment.index.column.BaseVirtualColumnProvider;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.index.readers.DocIdDictionary;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/VirtualColumnProvider.java
index 2ad70e3..658a4a4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/VirtualColumnProvider.java
@@ -18,8 +18,8 @@
*/
package org.apache.pinot.segment.local.segment.virtualcolumn;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTree.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTree.java
index ddd39c2..c700963 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTree.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTree.java
@@ -25,10 +25,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.startree.StarTree;
import org.apache.pinot.segment.spi.index.startree.StarTreeNode;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTreeNode.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTreeNode.java
index 58861ed..80e8fa6 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTreeNode.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/OffHeapStarTreeNode.java
@@ -19,8 +19,8 @@
package org.apache.pinot.segment.local.startree;
import java.util.Iterator;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.startree.StarTreeNode;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import static org.apache.pinot.segment.local.startree.StarTreeBuilderUtils.INVALID_ID;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
index b8ce913..a991feb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
@@ -31,13 +31,13 @@ import javax.annotation.Nullable;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
index 13b3dd7..31e6e2c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/MultipleTreesBuilder.java
@@ -32,16 +32,16 @@ import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.local.startree.StarTreeBuilderUtils;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils.IndexKey;
import org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils.IndexValue;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants.MetadataKey;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.ReadMode;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
index e4d2553..39ecf6f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
@@ -31,9 +31,9 @@ import java.util.Iterator;
import java.util.List;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexCombiner.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexCombiner.java
index a2319ac..3f3e281 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexCombiner.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeIndexCombiner.java
@@ -27,7 +27,7 @@ import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
index e98b735..e828506 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java
@@ -27,9 +27,9 @@ import java.util.Objects;
import java.util.Set;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.AggregationFunctionType;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
index 5acb9ca..b243a9f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeIndexContainer.java
@@ -25,11 +25,11 @@ import java.nio.ByteOrder;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.ReadMode;
import static org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils.IndexKey;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java
index 4613bc3..e03b267 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java
@@ -25,20 +25,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.pinot.segment.local.aggregator.ValueAggregatorFactory;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.forward.BaseChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReaderV2;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedByteChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.local.startree.OffHeapStarTree;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.segment.spi.index.startree.StarTree;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.MetricFieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CrcUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CrcUtils.java
index 4139b80..9dfd32f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CrcUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CrcUtils.java
@@ -30,7 +30,7 @@ import java.util.Collections;
import java.util.List;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
index ae6ebb0..351a09a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
@@ -38,7 +38,7 @@ import org.apache.pinot.common.exception.HttpErrorStatusException;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.spi.filesystem.PinotFS;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.apache.pinot.spi.ingestion.batch.spec.Constants;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/PinotBufferIndexInput.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/PinotBufferIndexInput.java
index d93b347..3fdf48e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/PinotBufferIndexInput.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/PinotBufferIndexInput.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.utils.fst;
import java.io.IOException;
import org.apache.lucene.store.IndexInput;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java
index 99a2bab..f30f340 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReaderTest.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetTest.java
index b5be91d..805f88a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetTest.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.io.util;
import java.io.IOException;
import java.nio.ByteOrder;
import java.util.Random;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2Test.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2Test.java
index 8944d47..442249c 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2Test.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/PinotDataBitSetV2Test.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.io.util;
import java.nio.ByteOrder;
import java.util.Random;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/VarLengthValueReaderWriterTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/VarLengthValueReaderWriterTest.java
index 8d551ba..32d8912 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/VarLengthValueReaderWriterTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/util/VarLengthValueReaderWriterTest.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.StringUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManagerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManagerTest.java
index 43a743d..2804cb3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManagerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/io/writer/impl/MmapMemoryManagerTest.java
@@ -25,7 +25,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.readerwriter.PinotDataBufferMemoryManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/BitmapInvertedIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/BitmapInvertedIndexCreatorTest.java
index 7bc8d42..790492d 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/BitmapInvertedIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/BitmapInvertedIndexCreatorTest.java
@@ -24,11 +24,11 @@ import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.OffHeapBitmapInvertedIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.inv.OnHeapBitmapInvertedIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.roaringbitmap.IntIterator;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/IntArraysTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/IntArraysTest.java
index 93d33e0..94e96f5 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/IntArraysTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/IntArraysTest.java
@@ -28,11 +28,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.spi.utils.ReadMode;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
index 21219a1..ef65b23 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.pinot.segment.spi.V1Constants;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
index b58c933..88fd1ba 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/ColumnMetadataTest.java
@@ -24,11 +24,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
index dd7b7b5..6b25b5e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
@@ -25,16 +25,16 @@ import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.realtime.impl.geospatial.MutableH3Index;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.OffHeapH3IndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.OnHeapH3IndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.geospatial.ImmutableH3IndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.local.utils.GeometryUtils;
import org.apache.pinot.segment.local.utils.H3Utils;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.GeoSpatialIndexCreator;
import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Point;
import org.testng.Assert;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
index e24f1bc..727d9f8 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
@@ -22,13 +22,13 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.realtime.impl.json.MutableJsonIndex;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OffHeapJsonIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OnHeapJsonIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.json.ImmutableJsonIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
index 327122c..d559923 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/SegmentMetadataImplTest.java
@@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
index b01e40c..2f9cd91 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
@@ -18,13 +18,13 @@
*/
package org.apache.pinot.segment.local.segment.index.column;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueDoubleDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueFloatDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueIntDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueLongDictionary;
import org.apache.pinot.segment.local.segment.index.readers.ConstantValueStringDictionary;
import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverterTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverterTest.java
index 6d7ab3a..38beff3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverterTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverterTest.java
@@ -26,14 +26,14 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BloomFilterCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BloomFilterCreatorTest.java
index 6e832a1..f531a42 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BloomFilterCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BloomFilterCreatorTest.java
@@ -20,12 +20,12 @@ package org.apache.pinot.segment.local.segment.index.creator;
import java.io.File;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.creator.impl.bloom.OnHeapGuavaBloomFilterCreator;
import org.apache.pinot.segment.local.segment.index.readers.bloom.BloomFilterReaderFactory;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.BloomFilterCreator;
import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
index ac58d13..128411d 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
@@ -24,13 +24,13 @@ import java.nio.ByteOrder;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.inv.text.LuceneFSTIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.LuceneFSTIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
public class LuceneFSTIndexCreatorTest {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RangeIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RangeIndexCreatorTest.java
index 8abab33..a3289e6 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RangeIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RangeIndexCreatorTest.java
@@ -24,7 +24,7 @@ import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.inv.RangeIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.RangeIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -34,7 +34,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION;
public class RangeIndexCreatorTest {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
index 490c1bb..9f515e8 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
@@ -23,19 +23,22 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.pinot.common.utils.StringUtil;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.index.readers.forward.BaseChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedByteChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
@@ -44,6 +47,7 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
@@ -231,7 +235,10 @@ public class RawIndexCreatorTest {
SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
driver.init(config, recordReader);
driver.build();
- _segmentDirectory = SegmentDirectory.createFromLocalFS(driver.getOutputDirectory(), ReadMode.mmap);
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ _segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(driver.getOutputDirectory().toURI(), new PinotConfiguration(props));
_segmentReader = _segmentDirectory.createReader();
recordReader.rewind();
return recordReader;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
index 7995f55..75c821a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
@@ -35,15 +36,16 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
import org.apache.pinot.segment.local.aggregator.PercentileTDigestValueAggregator;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.index.readers.BaseImmutableDictionary;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
@@ -52,6 +54,7 @@ import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordReader;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.ReadMode;
@@ -266,7 +269,10 @@ public class SegmentGenerationWithBytesTypeTest {
driver.init(config, recordReader);
driver.build();
- SegmentDirectory.createFromLocalFS(driver.getOutputDirectory(), ReadMode.mmap);
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(driver.getOutputDirectory().toURI(), new PinotConfiguration(props));
recordReader.rewind();
return recordReader;
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithFilterRecordsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithFilterRecordsTest.java
index e55578b..2dd520f 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithFilterRecordsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithFilterRecordsTest.java
@@ -24,11 +24,10 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.ingestion.FilterConfig;
@@ -84,7 +83,7 @@ public class SegmentGenerationWithFilterRecordsTest {
public void testNumDocs()
throws Exception {
File segmentDir = buildSegment(_tableConfig, _schema);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getTotalDocs(), 2);
PinotSegmentRecordReader segmentRecordReader = new PinotSegmentRecordReader(segmentDir);
GenericRow next = segmentRecordReader.next();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMultipleRecordsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMultipleRecordsTest.java
index b508a01..8b598f8 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMultipleRecordsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMultipleRecordsTest.java
@@ -27,10 +27,9 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
@@ -72,7 +71,7 @@ public class SegmentGenerationWithMultipleRecordsTest {
public void testNumDocs()
throws Exception {
File segmentDir = buildSegment(_tableConfig, _schema);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getTotalDocs(), 6);
Assert.assertTrue(metadata.getAllColumns().containsAll(Sets.newHashSet(SUB_COLUMN_1, SUB_COLUMN_2)));
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithNoRecordsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithNoRecordsTest.java
index 79fadce..8c427a5 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithNoRecordsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithNoRecordsTest.java
@@ -24,11 +24,10 @@ import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
@@ -92,7 +91,7 @@ public class SegmentGenerationWithNoRecordsTest {
public void testNumDocs()
throws Exception {
File segmentDir = buildSegment(_tableConfig, _schema);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getTotalDocs(), 0);
Assert.assertEquals(metadata.getTimeColumn(), DATE_TIME_COLUMN);
Assert.assertEquals(metadata.getTimeUnit(), TimeUnit.MILLISECONDS);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithTimeColumnTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithTimeColumnTest.java
index 3f7bdf0..3a550da 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithTimeColumnTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithTimeColumnTest.java
@@ -27,10 +27,9 @@ import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
@@ -87,7 +86,7 @@ public class SegmentGenerationWithTimeColumnTest {
throws Exception {
Schema schema = createSchema(true);
File segmentDir = buildSegment(_tableConfig, schema, true, false);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getStartTime(), sdfToMillis(minTime));
Assert.assertEquals(metadata.getEndTime(), sdfToMillis(maxTime));
}
@@ -100,7 +99,7 @@ public class SegmentGenerationWithTimeColumnTest {
throws Exception {
Schema schema = createDateTimeFieldSpecSchema(true);
File segmentDir = buildSegment(_tableConfig, schema, true, false);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getStartTime(), sdfToMillis(minTime));
Assert.assertEquals(metadata.getEndTime(), sdfToMillis(maxTime));
}
@@ -110,7 +109,7 @@ public class SegmentGenerationWithTimeColumnTest {
throws Exception {
Schema schema = createSchema(false);
File segmentDir = buildSegment(_tableConfig, schema, false, false);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getStartTime(), minTime);
Assert.assertEquals(metadata.getEndTime(), maxTime);
}
@@ -123,7 +122,7 @@ public class SegmentGenerationWithTimeColumnTest {
throws Exception {
Schema schema = createDateTimeFieldSpecSchema(false);
File segmentDir = buildSegment(_tableConfig, schema, false, false);
- SegmentMetadataImpl metadata = SegmentDirectory.loadSegmentMetadata(segmentDir);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
Assert.assertEquals(metadata.getStartTime(), minTime);
Assert.assertEquals(metadata.getEndTime(), maxTime);
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentPartitionTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentPartitionTest.java
index 090975c..c693704 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentPartitionTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentPartitionTest.java
@@ -31,13 +31,13 @@ import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.request.FilterOperator;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.segment.local.partition.ModuloPartitionFunction;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.partition.ModuloPartitionFunction;
import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
index 82c87ff..4cee3cc 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
@@ -23,9 +23,9 @@ import java.io.IOException;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitMVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitMVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedByteChunkSVForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedByteChunkSVForwardIndexTest.java
index a40eab5..a40bdfb 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedByteChunkSVForwardIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedByteChunkSVForwardIndexTest.java
@@ -26,8 +26,8 @@ import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWrit
import org.apache.pinot.segment.local.io.writer.impl.FixedByteChunkSVForwardIndexWriter;
import org.apache.pinot.segment.local.segment.index.readers.forward.BaseChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.FixedByteChunkSVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/SortedForwardIndexReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/SortedForwardIndexReaderTest.java
index c13928c..a9eb289 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/SortedForwardIndexReaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/SortedForwardIndexReaderTest.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.util.Random;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
import org.apache.pinot.segment.local.segment.index.readers.sorted.SortedIndexReaderImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/VarByteChunkSVForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/VarByteChunkSVForwardIndexTest.java
index 98b9bb7..5ab8b84 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/VarByteChunkSVForwardIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/VarByteChunkSVForwardIndexTest.java
@@ -30,10 +30,10 @@ import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkSVForwardIndexW
import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueVarByteRawIndexCreator;
import org.apache.pinot.segment.local.segment.index.readers.forward.BaseChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.memory.PinotNativeOrderLBuffer;
-import org.apache.pinot.segment.local.segment.memory.PinotNonNativeOrderLBuffer;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotNativeOrderLBuffer;
+import org.apache.pinot.segment.spi.memory.PinotNonNativeOrderLBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index 6e95ae4..53a631b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -21,32 +21,36 @@ package org.apache.pinot.segment.local.segment.index.loader;
import java.io.File;
import java.net.URL;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
import org.apache.pinot.segment.local.segment.index.converter.SegmentV1V2ToV3FormatConverter;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.index.readers.StringDictionary;
-import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.CommonConstants.Segment.BuiltInVirtualColumn;
import org.apache.pinot.spi.utils.ReadMode;
@@ -57,7 +61,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import org.testng.collections.Lists;
-import static org.apache.pinot.segment.local.segment.creator.impl.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
public class LoaderTest {
@@ -74,6 +78,8 @@ public class LoaderTest {
private File _indexDir;
private IndexLoadingConfig _v1IndexLoadingConfig;
private IndexLoadingConfig _v3IndexLoadingConfig;
+ private SegmentDirectoryLoader _localSegmentDirectoryLoader;
+ private PinotConfiguration _pinotConfiguration;
@BeforeClass
public void setUp()
@@ -83,6 +89,9 @@ public class LoaderTest {
URL resourceUrl = getClass().getClassLoader().getResource(AVRO_DATA);
Assert.assertNotNull(resourceUrl);
_avroFile = new File(resourceUrl.getFile());
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.heap.toString());
+ _pinotConfiguration = new PinotConfiguration(props);
_v1IndexLoadingConfig = new IndexLoadingConfig();
_v1IndexLoadingConfig.setReadMode(ReadMode.mmap);
@@ -91,6 +100,8 @@ public class LoaderTest {
_v3IndexLoadingConfig = new IndexLoadingConfig();
_v3IndexLoadingConfig.setReadMode(ReadMode.mmap);
_v3IndexLoadingConfig.setSegmentVersion(SegmentVersion.v3);
+
+ _localSegmentDirectoryLoader = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader();
}
private Schema constructV1Segment()
@@ -187,7 +198,7 @@ public class LoaderTest {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.LEGACY_STRING_PAD_CHAR);
- SegmentDirectory segmentDir = SegmentDirectory.createFromLocalFS(segmentDirectory, segmentMetadata, ReadMode.heap);
+ SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
SegmentDirectory.Reader reader = segmentDir.createReader();
PinotDataBuffer dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
StringDictionary dict =
@@ -208,7 +219,7 @@ public class LoaderTest {
segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.LEGACY_STRING_PAD_CHAR);
- segmentDir = SegmentDirectory.createFromLocalFS(segmentDirectory, segmentMetadata, ReadMode.heap);
+ segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
reader = segmentDir.createReader();
dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
dict = new StringDictionary(dictionaryBuffer, columnMetadata.getCardinality(), columnMetadata.getColumnMaxLength(),
@@ -228,7 +239,7 @@ public class LoaderTest {
segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
- segmentDir = SegmentDirectory.createFromLocalFS(segmentDirectory, segmentMetadata, ReadMode.heap);
+ segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
reader = segmentDir.createReader();
dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
dict = new StringDictionary(dictionaryBuffer, columnMetadata.getCardinality(), columnMetadata.getColumnMaxLength(),
@@ -313,7 +324,7 @@ public class LoaderTest {
// check that index dir is not in V1 format (the only subdir it should have is V3)
verifyIndexDirIsV3(_indexDir);
- SegmentDirectory segmentDir = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.heap);
+ SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
SegmentDirectory.Reader reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
@@ -329,7 +340,7 @@ public class LoaderTest {
Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(_indexDir, SegmentVersion.v3).exists());
// check that index dir is not in V1 format (the only subdir it should have is V3)
verifyIndexDirIsV3(_indexDir);
- segmentDir = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.heap);
+ segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
@@ -403,7 +414,7 @@ public class LoaderTest {
verifyIndexDirIsV3(_indexDir);
fstIndexFile = SegmentDirectoryPaths.findFSTIndexIndexFile(_indexDir, FST_INDEX_COL_NAME);
Assert.assertNull(fstIndexFile);
- segmentDir = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.heap);
+ segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
@@ -448,7 +459,7 @@ public class LoaderTest {
Assert.assertNotNull(textIndexFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
// CASE 1: don't set the segment version to load in IndexLoadingConfig
@@ -474,10 +485,10 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert
.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -502,10 +513,10 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert
.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -529,7 +540,7 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
// CASE 1: don't set the segment version to load in IndexLoadingConfig
@@ -552,10 +563,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -578,10 +589,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -604,10 +615,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert
.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index d6762fa..b96744a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -24,25 +24,29 @@ import java.nio.file.Files;
import java.nio.file.attribute.FileTime;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.segment.local.loader.LocalSegmentDirectoryLoader;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
-import org.apache.pinot.segment.local.segment.creator.TextIndexType;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.local.segment.index.converter.SegmentV1V2ToV3FormatConverter;
import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.ColumnIndexType;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.creator.TextIndexType;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
@@ -50,6 +54,7 @@ import org.apache.pinot.spi.config.table.ingestion.TransformConfig;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.ReadMode;
@@ -98,6 +103,7 @@ public class SegmentPreProcessorTest {
private static final String NEW_TDIGEST_BYTE_METRIC_COLUMN_NAME = "newTDigestByteMetric";
private File _indexDir;
+ private PinotConfiguration _configuration;
private IndexLoadingConfig _indexLoadingConfig;
private File _avroFile;
private Schema _schema;
@@ -113,6 +119,10 @@ public class SegmentPreProcessorTest {
throws Exception {
FileUtils.deleteQuietly(INDEX_DIR);
+ Map<String, Object> props = new HashMap<>();
+ props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
+ _configuration = new PinotConfiguration(props);
+
// We specify two columns without inverted index ('column1', 'column13'), one non-existing column ('noSuchColumn')
// and one column with existed inverted index ('column7').
_indexLoadingConfig = new IndexLoadingConfig();
@@ -218,11 +228,15 @@ public class SegmentPreProcessorTest {
_indexLoadingConfig.setFSTIndexColumns(fstColumns);
_indexLoadingConfig.getNoDictionaryColumns().add(EXISTING_STRING_COL_RAW);
constructV3Segment();
- SegmentPreProcessor v3Processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, _newColumnsSchema4);
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor v3Processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchema4);
Assert.expectThrows(UnsupportedOperationException.class, () -> v3Processor.process());
constructV1Segment();
- SegmentPreProcessor v1Processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, _newColumnsSchema4);
+ segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor v1Processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchema4);
Assert.expectThrows(UnsupportedOperationException.class, () -> v1Processor.process());
}
@@ -378,7 +392,11 @@ public class SegmentPreProcessorTest {
private ColumnMetadata checkIndexCreation(ColumnIndexType indexType, String column, int cardinality, int bits,
Schema schema, boolean isAutoGenerated, boolean hasDictionary, boolean isSorted, int dictionaryElementSize)
throws Exception {
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, schema)) {
+
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, schema)) {
processor.process();
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir);
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
@@ -398,8 +416,10 @@ public class SegmentPreProcessorTest {
Assert.assertEquals(columnMetadata.isAutoGenerated(), isAutoGenerated);
Assert.assertEquals(columnMetadata.getDefaultNullValueString(), "null");
- try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.mmap);
- SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ try (
+ SegmentDirectory segmentDirectory1 = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentDirectory.Reader reader = segmentDirectory1.createReader()) {
Assert.assertTrue(reader.hasIndexFor(column, indexType));
Assert.assertTrue(reader.hasIndexFor(column, ColumnIndexType.FORWARD_INDEX));
// if the text index is enabled on a new column with dictionary,
@@ -484,7 +504,9 @@ public class SegmentPreProcessorTest {
// Create inverted index the first time.
checkInvertedIndexCreation(false);
long addedLength = 0L;
- try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(segmentDirectoryPath, ReadMode.mmap);
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
// 8 bytes overhead is for checking integrity of the segment.
addedLength += reader.getIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX).size() + 8;
@@ -506,7 +528,9 @@ public class SegmentPreProcessorTest {
private void checkInvertedIndexCreation(boolean reCreate)
throws Exception {
- try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.mmap);
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
if (reCreate) {
Assert.assertTrue(reader.hasIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX));
@@ -521,11 +545,16 @@ public class SegmentPreProcessorTest {
}
}
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, null)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
- try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.mmap);
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
Assert.assertTrue(reader.hasIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX));
Assert.assertTrue(reader.hasIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX));
@@ -546,7 +575,10 @@ public class SegmentPreProcessorTest {
// Try to use the third schema and update default value again.
// For the third schema, we changed the default value for column 'newStringMVDimension' to 'notSameLength', which
// is not the same length as before. This should be fine for segment format v1.
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, _newColumnsSchema3)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchema3)) {
processor.process();
}
@@ -573,7 +605,10 @@ public class SegmentPreProcessorTest {
private void checkUpdateDefaultColumns()
throws Exception {
// Update default value.
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, _newColumnsSchema1)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchema1)) {
processor.process();
}
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -649,7 +684,9 @@ public class SegmentPreProcessorTest {
Assert.assertEquals(columnMetadata.getMaxValue(), (int) originalColumnMetadata.getMaxValue() + 1);
// Check dictionary and forward index exist.
- try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(_indexDir, ReadMode.mmap);
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
Assert.assertTrue(reader.hasIndexFor(NEW_INT_METRIC_COLUMN_NAME, ColumnIndexType.DICTIONARY));
Assert.assertTrue(reader.hasIndexFor(NEW_INT_METRIC_COLUMN_NAME, ColumnIndexType.FORWARD_INDEX));
@@ -670,7 +707,10 @@ public class SegmentPreProcessorTest {
// Use the second schema and update default value again.
// For the second schema, we changed the default value for column 'newIntMetric' to 2, and added default value
// 'abcd' (keep the same length as 'null') to column 'newStringMVDimension'.
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, _indexLoadingConfig, _newColumnsSchema2)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchema2)) {
processor.process();
}
segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -706,7 +746,10 @@ public class SegmentPreProcessorTest {
IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NONE);
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -721,7 +764,10 @@ public class SegmentPreProcessorTest {
Assert.assertNull(metricColumnMetadata.getMaxValue());
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.TIME);
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -736,7 +782,10 @@ public class SegmentPreProcessorTest {
Assert.assertNull(metricColumnMetadata.getMaxValue());
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NON_METRIC);
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -751,7 +800,10 @@ public class SegmentPreProcessorTest {
Assert.assertNull(metricColumnMetadata.getMaxValue());
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.ALL);
- try (SegmentPreProcessor processor = new SegmentPreProcessor(_indexDir, indexLoadingConfig, null)) {
+ try (
+ SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), _configuration);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
segmentMetadata = new SegmentMetadataImpl(_indexDir);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
index 2b1afd4..2c1cb9e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
@@ -28,10 +28,11 @@ import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
import org.apache.pinot.segment.local.segment.index.SegmentMetadataImplTest;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectory;
+import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.ReadMode;
@@ -74,7 +75,7 @@ public class BaseDefaultColumnHandlerTest {
driver.build();
segmentDirectory = new File(INDEX_DIR, driver.getSegmentName());
committedSegmentMetadata = new SegmentMetadataImpl(segmentDirectory);
- writer = SegmentDirectory.createFromLocalFS(INDEX_DIR, committedSegmentMetadata, ReadMode.mmap).createWriter();
+ writer = new SegmentLocalFSDirectory(INDEX_DIR, committedSegmentMetadata, ReadMode.mmap).createWriter();
}
@AfterMethod
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/reader/FixedByteWidthRowColDataFileReaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/reader/FixedByteWidthRowColDataFileReaderTest.java
index ad3027b..5897c57 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/reader/FixedByteWidthRowColDataFileReaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/reader/FixedByteWidthRowColDataFileReaderTest.java
@@ -24,7 +24,7 @@ import java.io.FileOutputStream;
import java.io.RandomAccessFile;
import java.util.Random;
import org.apache.pinot.segment.local.io.reader.impl.FixedByteSingleValueMultiColReader;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
index 3559a09..decd434 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
@@ -30,8 +30,8 @@ import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.ByteArray;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTypeConversionTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTypeConversionTest.java
index 3eec1cd..2c2e1dc 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTypeConversionTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTypeConversionTest.java
@@ -24,8 +24,8 @@ import java.util.Arrays;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.ArrayCopyUtils;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImplTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImplTest.java
index 0934d0a..3bc7db2 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImplTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/NullValueVectorReaderImplTest.java
@@ -22,8 +22,8 @@ import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java
index db083d9..07cc843 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitSVForwardIndexReaderV2Test.java
@@ -24,7 +24,7 @@ import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.util.PinotDataBitSetV2;
import org.apache.pinot.segment.local.io.writer.impl.FixedBitSVForwardIndexWriter;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/writer/FixedByteWidthRowColForwardIndexWriterTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/writer/FixedByteWidthRowColForwardIndexWriterTest.java
index 4641fe5..df53e33 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/writer/FixedByteWidthRowColForwardIndexWriterTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/writer/FixedByteWidthRowColForwardIndexWriterTest.java
@@ -25,8 +25,8 @@ import java.util.Random;
import org.apache.commons.lang.StringUtils;
import org.apache.pinot.segment.local.io.reader.impl.FixedByteSingleValueMultiColReader;
import org.apache.pinot.segment.local.io.writer.impl.FixedByteSingleValueMultiColWriter;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectoryTestHelper.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectoryTestHelper.java
index c981d00..02fc463 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectoryTestHelper.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectoryTestHelper.java
@@ -19,9 +19,11 @@
package org.apache.pinot.segment.local.segment.store;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index 4a6e119..1abb263 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -21,9 +21,11 @@ package org.apache.pinot.segment.local.segment.store;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPathsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPathsTest.java
index 64ad834..400ba80 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPathsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPathsTest.java
@@ -20,8 +20,9 @@ package org.apache.pinot.segment.local.segment.store;
import java.io.File;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java
index fa2ec41..1781b51 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SegmentLocalFSDirectoryTest.java
@@ -19,11 +19,13 @@
package org.apache.pinot.segment.local.segment.store;
import java.io.File;
-import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.utils.ReadMode;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -33,11 +35,12 @@ import org.testng.annotations.Test;
public class SegmentLocalFSDirectoryTest {
private static final File TEST_DIRECTORY = new File(SingleFileIndexDirectoryTest.class.toString());
- SegmentLocalFSDirectory segmentDirectory;
+ SegmentDirectory segmentDirectory;
SegmentMetadataImpl metadata;
@BeforeClass
- public void setUp() {
+ public void setUp()
+ throws Exception {
FileUtils.deleteQuietly(TEST_DIRECTORY);
TEST_DIRECTORY.mkdirs();
metadata = ColumnIndexDirectoryTestHelper.writeMetadata(SegmentVersion.v1);
@@ -54,12 +57,12 @@ public class SegmentLocalFSDirectoryTest {
@Test
public void testMultipleReadersNoWriter()
throws Exception {
- SegmentLocalFSDirectory.Reader reader = segmentDirectory.createReader();
+ SegmentDirectory.Reader reader = segmentDirectory.createReader();
Assert.assertNotNull(reader);
- SegmentLocalFSDirectory.Reader reader1 = segmentDirectory.createReader();
+ SegmentDirectory.Reader reader1 = segmentDirectory.createReader();
Assert.assertNotNull(reader1);
- SegmentLocalFSDirectory.Writer writer = segmentDirectory.createWriter();
+ SegmentDirectory.Writer writer = segmentDirectory.createWriter();
Assert.assertNull(writer);
reader.close();
reader1.close();
@@ -68,13 +71,13 @@ public class SegmentLocalFSDirectoryTest {
@Test
public void testExclusiveWrite()
throws java.lang.Exception {
- SegmentLocalFSDirectory.Writer writer = segmentDirectory.createWriter();
+ SegmentDirectory.Writer writer = segmentDirectory.createWriter();
Assert.assertNotNull(writer);
- SegmentLocalFSDirectory.Reader reader2 = segmentDirectory.createReader();
+ SegmentDirectory.Reader reader2 = segmentDirectory.createReader();
Assert.assertNull(reader2);
- SegmentLocalFSDirectory.Writer writer1 = segmentDirectory.createWriter();
+ SegmentDirectory.Writer writer1 = segmentDirectory.createWriter();
Assert.assertNull(writer1);
writer.close();
@@ -100,7 +103,7 @@ public class SegmentLocalFSDirectoryTest {
@Test
public void testWriteAndReadBackData()
throws java.lang.Exception {
- try (SegmentLocalFSDirectory.Writer writer = segmentDirectory.createWriter()) {
+ try (SegmentDirectory.Writer writer = segmentDirectory.createWriter()) {
Assert.assertNotNull(writer);
PinotDataBuffer buffer = writer.newIndexFor("newColumn", ColumnIndexType.FORWARD_INDEX, 1024);
loadData(buffer);
@@ -115,27 +118,27 @@ public class SegmentLocalFSDirectoryTest {
@Test
public void testDirectorySize()
- throws IOException {
+ throws Exception {
// this test verifies that the segment size is returned correctly even if v3/ subdir
// does not exist. We have not good way to test all the conditions since the
// format converters are higher level modules that can not be used in this package
// So, we do what we can do best....HACK HACK HACK
File sizeTestDirectory = null;
+
try {
sizeTestDirectory = new File(SegmentLocalFSDirectoryTest.class.getName() + "-size_test");
if (sizeTestDirectory.exists()) {
FileUtils.deleteQuietly(sizeTestDirectory);
}
FileUtils.copyDirectoryToDirectory(segmentDirectory.getPath().toFile(), sizeTestDirectory);
- SegmentDirectory sizeSegment =
- SegmentLocalFSDirectory.createFromLocalFS(sizeTestDirectory, metadata, ReadMode.mmap);
+ SegmentDirectory sizeSegment = new SegmentLocalFSDirectory(sizeTestDirectory, metadata, ReadMode.mmap);
Assert.assertEquals(sizeSegment.getDiskSizeBytes(), segmentDirectory.getDiskSizeBytes());
Assert.assertFalse(SegmentDirectoryPaths.segmentDirectoryFor(sizeTestDirectory, SegmentVersion.v3).exists());
File v3SizeDir = new File(sizeTestDirectory, SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
// the destination is not exactly v3 but does not matter
FileUtils.copyDirectoryToDirectory(segmentDirectory.getPath().toFile(), v3SizeDir);
- SegmentDirectory sizeV3Segment = SegmentDirectory.createFromLocalFS(v3SizeDir, metadata, ReadMode.mmap);
+ SegmentDirectory sizeV3Segment = new SegmentLocalFSDirectory(v3SizeDir, metadata, ReadMode.mmap);
Assert.assertEquals(sizeSegment.getDiskSizeBytes(), sizeV3Segment.getDiskSizeBytes());
// now drop v3
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 36d0ddd..1c3f806 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -24,9 +24,11 @@ import java.util.Arrays;
import java.util.HashSet;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.store.ColumnIndexDirectory;
+import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.util.TestUtils;
import org.mockito.Mockito;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
index 739b5e5..fc8e46f 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfigTest.java
@@ -21,10 +21,10 @@ package org.apache.pinot.segment.local.startree.v2.builder;
import java.util.Arrays;
import java.util.HashSet;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.AggregationFunctionType;
import org.apache.pinot.segment.spi.Constants;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
index a8187aa..5315e7b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
@@ -31,7 +31,7 @@ import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.OffHeapFSTStore;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.PositiveIntOutputs;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
diff --git a/pinot-segment-spi/pom.xml b/pinot-segment-spi/pom.xml
index 10a2e6f..8b37512 100644
--- a/pinot-segment-spi/pom.xml
+++ b/pinot-segment-spi/pom.xml
@@ -72,6 +72,10 @@
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.xerial.larray</groupId>
+ <artifactId>larray-mmap</artifactId>
+ </dependency>
<!-- test -->
<dependency>
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
similarity index 96%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/V1Constants.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 8c7059a..cf546cc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.creator.impl;
+package org.apache.pinot.segment.spi;
public class V1Constants {
public static final String SEGMENT_CREATION_META = "creation.meta";
@@ -42,6 +42,8 @@ public class V1Constants {
public static final String H3_INDEX_FILE_EXTENSION = ".h3.idx";
public static final String BLOOM_FILTER_FILE_EXTENSION = ".bloom";
public static final String NULLVALUE_VECTOR_FILE_EXTENSION = ".bitmap.nullvalue";
+ public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
+ public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
}
public static class MetadataKeys {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/TextIndexType.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java
similarity index 93%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/TextIndexType.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java
index 8943fa1..345e47b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/TextIndexType.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/TextIndexType.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.creator;
+package org.apache.pinot.segment.spi.index.creator;
public enum TextIndexType {
NONE,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/ColumnMetadata.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadata.java
similarity index 98%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/ColumnMetadata.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadata.java
index 7ebe8fa..b8859f4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/ColumnMetadata.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadata.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.index.metadata;
+package org.apache.pinot.segment.spi.index.metadata;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.lang.reflect.Field;
@@ -24,11 +24,11 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.pinot.common.metadata.segment.ColumnPartitionMetadata;
-import org.apache.pinot.segment.local.partition.PartitionFunctionFactory;
-import org.apache.pinot.segment.local.segment.creator.TextIndexType;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.creator.TextIndexType;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
+import org.apache.pinot.segment.spi.partition.metadata.ColumnPartitionMetadata;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/SegmentMetadataImpl.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
similarity index 95%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/SegmentMetadataImpl.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
index be064cd..ad6bcbd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/metadata/SegmentMetadataImpl.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.index.metadata;
+package org.apache.pinot.segment.spi.index.metadata;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -44,13 +44,12 @@ import javax.annotation.Nullable;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.store.SegmentDirectoryPaths;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.CommonsConfigurationUtils;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -120,32 +119,32 @@ public class SegmentMetadataImpl implements SegmentMetadata {
/**
* For REALTIME consuming segments.
*/
- public SegmentMetadataImpl(RealtimeSegmentZKMetadata segmentMetadata, Schema schema) {
+ public SegmentMetadataImpl(String rawTableName, String segmentName, long creationTime, long startTime, long endTime,
+ @Nullable TimeUnit timeUnit, long totalDocs, long crc, Schema schema) {
_indexDir = null;
PropertiesConfiguration segmentMetadataPropertiesConfiguration = new PropertiesConfiguration();
segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_CREATOR_VERSION, null);
segmentMetadataPropertiesConfiguration
.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_PADDING_CHARACTER, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
segmentMetadataPropertiesConfiguration
- .addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME, Long.toString(segmentMetadata.getStartTime()));
- segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME, Long.toString(segmentMetadata.getEndTime()));
- segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.TABLE_NAME, segmentMetadata.getTableName());
+ .addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_START_TIME, Long.toString(startTime));
+ segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_END_TIME, Long.toString(endTime));
+ segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.TABLE_NAME, rawTableName);
- TimeUnit timeUnit = segmentMetadata.getTimeUnit();
if (timeUnit != null) {
segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.TIME_UNIT, timeUnit.toString());
} else {
segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.TIME_UNIT, null);
}
- segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS, segmentMetadata.getTotalDocs());
+ segmentMetadataPropertiesConfiguration.addProperty(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS, totalDocs);
- _crc = segmentMetadata.getCrc();
- _creationTime = segmentMetadata.getCreationTime();
+ _crc = crc;
+ _creationTime = creationTime;
setTimeInfo(segmentMetadataPropertiesConfiguration);
_columnMetadataMap = null;
- _rawTableName = segmentMetadata.getTableName();
- _segmentName = segmentMetadata.getSegmentName();
+ _rawTableName = rawTableName;
+ _segmentName = segmentName;
_schema = schema;
_totalDocs = segmentMetadataPropertiesConfiguration.getInt(V1Constants.MetadataKeys.Segment.SEGMENT_TOTAL_DOCS);
_customMap = new HashMap<>();
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
similarity index 51%
copy from pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
copy to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
index eebcecd..337ae81 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
@@ -16,42 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.spi.config.instance;
+package org.apache.pinot.segment.spi.loader;
+import java.net.URI;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.ReadMode;
-public interface InstanceDataManagerConfig {
- PinotConfiguration getConfig();
-
- String getInstanceId();
-
- String getInstanceDataDir();
-
- String getConsumerDir();
-
- String getInstanceSegmentTarDir();
-
- String getInstanceBootstrapSegmentDir();
-
- String getSegmentStoreUri();
-
- ReadMode getReadMode();
-
- String getSegmentFormatVersion();
-
- String getAvgMultiValueCount();
-
- boolean isEnableSplitCommit();
-
- boolean isEnableSplitCommitEndWithMetadata();
-
- boolean isRealtimeOffHeapAllocation();
-
- boolean isDirectRealtimeOffHeapAllocation();
-
- int getMaxParallelSegmentBuilds();
-
- String getAuthToken();
+/**
+ * Interface for creating and loading the {@link SegmentDirectory} instance using provided config
+ */
+public interface SegmentDirectoryLoader {
+
+ /**
+ * Creates the {@link SegmentDirectory} instance
+ * @param indexDir index directory
+ * @param segmentDirectoryConfig config for SegmentDirectory, containing all properties needed to instantiate the
+ * {@link SegmentDirectory}
+ * e.g. readMode (MMAP/HEAP) or
+ * properties specific to the tier backend (deep store access configs)
+ */
+ SegmentDirectory load(URI indexDir, PinotConfiguration segmentDirectoryConfig)
+ throws Exception;
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderRegistry.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderRegistry.java
new file mode 100644
index 0000000..3b02593
--- /dev/null
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderRegistry.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.spi.loader;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.reflections.Reflections;
+import org.reflections.scanners.ResourcesScanner;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.scanners.TypeAnnotationsScanner;
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Helper class to dynamically register all annotated {@link SegmentLoader} methods
+ */
+public class SegmentDirectoryLoaderRegistry {
+ private static final String LOCAL_TIER_BACKEND_NAME = "local";
+ private SegmentDirectoryLoaderRegistry() {
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SegmentDirectoryLoaderRegistry.class);
+ private static final Map<String, SegmentDirectoryLoader> _segmentDirectoryLoaderMap = new HashMap<>();
+
+ static {
+ Reflections reflections = new Reflections(
+ new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.pinot.segment"))
+ .filterInputsBy(new FilterBuilder.Include(".*\\.loader\\..*"))
+ .setScanners(new ResourcesScanner(), new TypeAnnotationsScanner(), new SubTypesScanner()));
+ Set<Class<?>> classes = reflections.getTypesAnnotatedWith(SegmentLoader.class);
+ classes.forEach(loaderClass -> {
+ SegmentLoader segmentLoaderAnnotation = loaderClass.getAnnotation(SegmentLoader.class);
+ if (segmentLoaderAnnotation.enabled()) {
+ if (segmentLoaderAnnotation.name().isEmpty()) {
+ LOGGER.error("Cannot register an unnamed SegmentDirectoryLoader for annotation {} ",
+ segmentLoaderAnnotation.toString());
+ } else {
+ String segmentLoaderName = segmentLoaderAnnotation.name();
+ SegmentDirectoryLoader segmentDirectoryLoader;
+ try {
+ segmentDirectoryLoader = (SegmentDirectoryLoader) loaderClass.newInstance();
+ _segmentDirectoryLoaderMap.putIfAbsent(segmentLoaderName, segmentDirectoryLoader);
+ } catch (Exception e) {
+ LOGGER.error(
+ String.format("Unable to register SegmentDirectoryLoader %s . Cannot instantiate.", segmentLoaderName),
+ e);
+ }
+ }
+ }
+ });
+ LOGGER.info("Initialized {} with {} segmentDirectoryLoaders: {}", SegmentDirectoryLoaderRegistry.class.getName(),
+ _segmentDirectoryLoaderMap.size(), _segmentDirectoryLoaderMap.keySet());
+ }
+
+ /**
+ * Returns the segment directory loader instance from instantiated map, for the given tier backend
+ */
+ public static SegmentDirectoryLoader getSegmentDirectoryLoader(String tierBackend) {
+ return _segmentDirectoryLoaderMap.get(tierBackend);
+ }
+
+ /**
+ * Returns the instance of 'localSegmentDirectoryLoader'
+ */
+ public static SegmentDirectoryLoader getLocalSegmentDirectoryLoader() {
+ return _segmentDirectoryLoaderMap.get(LOCAL_TIER_BACKEND_NAME);
+ }
+}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentLoader.java
similarity index 56%
copy from pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java
copy to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentLoader.java
index f08f1c0..dd2d3c4 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentLoader.java
@@ -16,27 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.partition;
+package org.apache.pinot.segment.spi.loader;
-import java.io.Serializable;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
/**
- * Interface for partition function.
+ * Annotation class for {@link SegmentDirectoryLoader} implementations. The corresponding classes
+ * are auto-registered during startup.
+ *
+ * NOTE:
+ * 1. The annotated class must be under the package of name 'org.apache.pinot.segment.*.loader'
+ * to be auto-registered.
*/
-public interface PartitionFunction extends Serializable {
-
- /**
- * Method to compute and return partition id for the given value.
- *
- * @param value Value for which to determine the partition id.
- * @return partition id for the value.
- */
- int getPartition(Object value);
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface SegmentLoader {
+ boolean enabled() default true;
- /**
- * Returns the total number of possible partitions.
- * @return Number of possible partitions.
- */
- int getNumPartitions();
+ String name() default "";
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/BasePinotLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java
similarity index 98%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/BasePinotLBuffer.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java
index 494b9a1..1b2fc91 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/BasePinotLBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import java.io.File;
import java.io.IOException;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CleanerUtil.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/CleanerUtil.java
similarity index 99%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CleanerUtil.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/CleanerUtil.java
index 295c62c..fa0ba05 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/CleanerUtil.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/CleanerUtil.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.utils;
+package org.apache.pinot.segment.spi.memory;
import java.io.IOException;
import java.lang.invoke.MethodHandle;
@@ -196,4 +196,4 @@ public final class CleanerUtil {
public interface BufferCleaner {
void freeBuffer(ByteBuffer b) throws IOException;
}
-}
\ No newline at end of file
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotByteBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotByteBuffer.java
similarity index 98%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotByteBuffer.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotByteBuffer.java
index 0f8c4d8..0f962f0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotByteBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotByteBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import java.io.File;
import java.io.IOException;
@@ -27,7 +27,6 @@ import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import javax.annotation.concurrent.ThreadSafe;
-import org.apache.pinot.segment.local.utils.CleanerUtil;
@ThreadSafe
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotDataBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
similarity index 99%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotDataBuffer.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
index bcb2773..4d3b523 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotDataBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import com.google.common.annotations.VisibleForTesting;
import java.io.Closeable;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNativeOrderLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java
similarity index 98%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNativeOrderLBuffer.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java
index aa33af2..5747a3d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNativeOrderLBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import java.io.File;
import java.io.IOException;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNonNativeOrderLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java
similarity index 98%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNonNativeOrderLBuffer.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java
index a3331e8..42d77b7 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/memory/PinotNonNativeOrderLBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import java.io.File;
import java.io.IOException;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ByteArrayPartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ByteArrayPartitionFunction.java
similarity index 93%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ByteArrayPartitionFunction.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ByteArrayPartitionFunction.java
index a95ed2b..3aac9de 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ByteArrayPartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ByteArrayPartitionFunction.java
@@ -16,11 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import com.google.common.base.Preconditions;
import java.util.Arrays;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/HashCodePartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/HashCodePartitionFunction.java
similarity index 93%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/HashCodePartitionFunction.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/HashCodePartitionFunction.java
index ac033d8..4ef556d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/HashCodePartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/HashCodePartitionFunction.java
@@ -16,10 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import com.google.common.base.Preconditions;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ModuloPartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ModuloPartitionFunction.java
similarity index 95%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ModuloPartitionFunction.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ModuloPartitionFunction.java
index 4d32f3c..9f9b09f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/ModuloPartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/ModuloPartitionFunction.java
@@ -16,10 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import com.google.common.base.Preconditions;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
/**
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/MurmurPartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
similarity index 63%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/MurmurPartitionFunction.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
index ac57b79..be773b3 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/MurmurPartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
@@ -16,13 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.pinot.common.utils.HashUtil;
-import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.spi.utils.StringUtils;
/**
@@ -44,7 +42,7 @@ public class MurmurPartitionFunction implements PartitionFunction {
@Override
public int getPartition(Object valueIn) {
String value = (valueIn instanceof String) ? (String) valueIn : valueIn.toString();
- return (murmur2(StringUtil.encodeUtf8(value)) & 0x7fffffff) % _numPartitions;
+ return (murmur2(StringUtils.encodeUtf8(value)) & 0x7fffffff) % _numPartitions;
}
@Override
@@ -66,6 +64,44 @@ public class MurmurPartitionFunction implements PartitionFunction {
*/
@VisibleForTesting
int murmur2(final byte[] data) {
- return HashUtil.murmur2(data);
+ int length = data.length;
+ int seed = 0x9747b28c;
+ // 'm' and 'r' are mixing constants generated offline.
+ // They're not really 'magic', they just happen to work well.
+ final int m = 0x5bd1e995;
+ final int r = 24;
+
+ // Initialize the hash to a random value
+ int h = seed ^ length;
+ int length4 = length / 4;
+
+ for (int i = 0; i < length4; i++) {
+ final int i4 = i * 4;
+ int k =
+ (data[i4 + 0] & 0xff) + ((data[i4 + 1] & 0xff) << 8) + ((data[i4 + 2] & 0xff) << 16) + ((data[i4 + 3] & 0xff)
+ << 24);
+ k *= m;
+ k ^= k >>> r;
+ k *= m;
+ h *= m;
+ h ^= k;
+ }
+
+ // Handle the last few bytes of the input array
+ switch (length % 4) {
+ case 3:
+ h ^= (data[(length & ~3) + 2] & 0xff) << 16;
+ case 2:
+ h ^= (data[(length & ~3) + 1] & 0xff) << 8;
+ case 1:
+ h ^= data[length & ~3] & 0xff;
+ h *= m;
+ }
+
+ h ^= h >>> 13;
+ h *= m;
+ h ^= h >>> 15;
+
+ return h;
}
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java
index f08f1c0..6c8e942 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunction.java
@@ -25,7 +25,6 @@ import java.io.Serializable;
* Interface for partition function.
*/
public interface PartitionFunction extends Serializable {
-
/**
* Method to compute and return partition id for the given value.
*
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/PartitionFunctionFactory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
similarity index 96%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/PartitionFunctionFactory.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
index 75ac113..174d450 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/partition/PartitionFunctionFactory.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nonnull;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
/**
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadata.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/metadata/ColumnPartitionMetadata.java
similarity index 99%
rename from pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadata.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/metadata/ColumnPartitionMetadata.java
index 537f01e..cf1eadd 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/ColumnPartitionMetadata.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/metadata/ColumnPartitionMetadata.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.metadata.segment;
+package org.apache.pinot.segment.spi.partition.metadata;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonParser;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexDirectory.java
similarity index 91%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectory.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexDirectory.java
index 76759d5..6a451d9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexDirectory.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexDirectory.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.store;
+package org.apache.pinot.segment.spi.store;
import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.utils.ReadMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
* Abstract class to map the columnar indices to their location on disk.
*
*/
-abstract class ColumnIndexDirectory implements Closeable {
+public abstract class ColumnIndexDirectory implements Closeable {
private static final Logger LOGGER = LoggerFactory.getLogger(ColumnIndexDirectory.class);
protected File segmentDirectory;
@@ -60,6 +60,10 @@ abstract class ColumnIndexDirectory implements Closeable {
this.readMode = readMode;
}
+ public void setMetadata(SegmentMetadataImpl metadata) {
+ this.metadata = metadata;
+ }
+
/**
* Allocation context for tracking memory
* @param f file for which memory is allocatedx
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexType.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexType.java
similarity index 96%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexType.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexType.java
index 428ea49..7571731 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/ColumnIndexType.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/ColumnIndexType.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.store;
+package org.apache.pinot.segment.spi.store;
public enum ColumnIndexType {
DICTIONARY("dictionary"),
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
similarity index 83%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectory.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
index f71c17e..27f68e0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectory.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectory.java
@@ -16,16 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.store;
+package org.apache.pinot.segment.spi.store;
import java.io.Closeable;
-import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Path;
import org.apache.commons.configuration.ConfigurationException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
-import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
/**
@@ -94,36 +93,15 @@ import org.apache.pinot.spi.utils.ReadMode;
*/
public abstract class SegmentDirectory implements Closeable {
- /**
- * Create segment directory from local file system
- * @param directory File object representing segment directory on disk
- * @param metadata segment metadata
- * @param readMode mmap vs heap ReadMode for data
- * @return segmentDirectory
- */
- // NOTE: this needs to be metadata impl to read all columns.
- // In future, we will have this class load metadata rather than
- // passing it in.
- public static SegmentDirectory createFromLocalFS(File directory, SegmentMetadataImpl metadata, ReadMode readMode) {
- return new SegmentLocalFSDirectory(directory, metadata, readMode);
- }
+ public abstract URI getIndexDir();
- public static SegmentDirectory createFromLocalFS(File directory, ReadMode readMode)
- throws IOException, ConfigurationException {
- return new SegmentLocalFSDirectory(directory, readMode);
- }
-
- public static SegmentMetadataImpl loadSegmentMetadata(File directory)
- throws IOException, ConfigurationException {
- return SegmentLocalFSDirectory.loadSegmentMetadata(directory);
- }
+ public abstract SegmentMetadataImpl getSegmentMetadata();
public abstract void reloadMetadata()
throws Exception;
/**
* Get the path/URL for the directory
- * @return
*/
public abstract Path getPath();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPaths.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
similarity index 88%
rename from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPaths.java
rename to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
index d2f5cc8..9b54128 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/SegmentDirectoryPaths.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
@@ -16,15 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.store;
+package org.apache.pinot.segment.spi.store;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.File;
import javax.annotation.Nullable;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
-import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import static org.apache.pinot.segment.spi.creator.SegmentVersion.v3;
@@ -83,7 +81,7 @@ public class SegmentDirectoryPaths {
*/
@Nullable
public static File findTextIndexIndexFile(File indexDir, String column) {
- String luceneIndexDirectory = column + LuceneTextIndexCreator.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
return findFormatFile(indexDir, luceneIndexDirectory);
}
@@ -96,7 +94,7 @@ public class SegmentDirectoryPaths {
@Nullable
@VisibleForTesting
public static File findTextIndexDocIdMappingFile(File indexDir, String column) {
- String file = column + LuceneTextIndexReader.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ String file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
return findFormatFile(indexDir, file);
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/memory/PinotDataBufferTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTest.java
similarity index 99%
rename from pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/memory/PinotDataBufferTest.java
rename to pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTest.java
index 7dd0f2b..1538351 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/memory/PinotDataBufferTest.java
+++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotDataBufferTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.memory;
+package org.apache.pinot.segment.spi.memory;
import java.io.File;
import java.io.RandomAccessFile;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/partition/PartitionFunctionTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
similarity index 97%
rename from pinot-segment-local/src/test/java/org/apache/pinot/segment/local/partition/PartitionFunctionTest.java
rename to pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
index c5fcb55..58efcf2 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/partition/PartitionFunctionTest.java
+++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
@@ -16,11 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.partition;
+package org.apache.pinot.segment.spi.partition;
import java.util.Random;
-import org.apache.pinot.common.utils.StringUtil;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.spi.utils.StringUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -256,7 +255,7 @@ public class PartitionFunctionTest {
byte[] array = new byte[7];
for (int expectedPartition : expectedPartitions) {
random.nextBytes(array);
- String nextString = StringUtil.decodeUtf8(array);
+ String nextString = StringUtils.decodeUtf8(array);
int actualPartition = partitionFunction.getPartition(nextString);
Assert.assertEquals(actualPartition, expectedPartition);
}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/MmapDebugResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/MmapDebugResource.java
index fa9497b..b925751 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/MmapDebugResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/MmapDebugResource.java
@@ -36,7 +36,7 @@ import javax.ws.rs.core.Response;
import org.apache.pinot.common.restlet.resources.ResourceUtils;
import org.apache.pinot.core.data.manager.InstanceDataManager;
import org.apache.pinot.core.data.manager.realtime.RealtimeTableDataManager;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.server.starter.ServerInstance;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
index 981d021..01f98f3 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
@@ -29,9 +29,9 @@ import java.util.Set;
import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentImpl;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.utils.JsonUtils;
/**
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index e26adb9..4067d72 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -58,7 +58,7 @@ import org.apache.pinot.core.data.manager.realtime.RealtimeSegmentDataManager;
import org.apache.pinot.core.data.manager.realtime.SegmentUploader;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.server.api.access.AccessControl;
import org.apache.pinot.server.api.access.AccessControlFactory;
import org.apache.pinot.server.starter.ServerInstance;
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
index 63a7333..e0da5fd 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.server.starter.helix;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Optional;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.pinot.spi.config.instance.InstanceDataManagerConfig;
@@ -63,6 +65,11 @@ public class HelixInstanceDataManagerConfig implements InstanceDataManagerConfig
public static final String INSTANCE_RELOAD_CONSUMING_SEGMENT = "reload.consumingSegment";
// Key of the auth token
public static final String AUTH_TOKEN = "auth.token";
+ // Tier properties
+ public static final String TIER_BACKEND = "tier.backend";
+ public static final String DEFAULT_TIER_BACKEND = "local";
+ // Prefix for tier config
+ public static final String TIER_CONFIGS_PREFIX = "tier";
// Key of how many parallel realtime segments can be built.
// A value of <= 0 indicates unlimited.
@@ -103,7 +110,7 @@ public class HelixInstanceDataManagerConfig implements InstanceDataManagerConfig
throws ConfigurationException {
_instanceDataManagerConfiguration = serverConfig;
- for (String key: serverConfig.getKeys()) {
+ for (String key : serverConfig.getKeys()) {
LOGGER.info("InstanceDataManagerConfig, key: {} , value: {}", key, serverConfig.getProperty(key));
}
@@ -208,6 +215,23 @@ public class HelixInstanceDataManagerConfig implements InstanceDataManagerConfig
}
@Override
+ public String getTierBackend() {
+ return _instanceDataManagerConfiguration.getProperty(TIER_BACKEND, DEFAULT_TIER_BACKEND);
+ }
+
+ @Override
+ public PinotConfiguration getTierConfigs() {
+ String tierBackend = getTierBackend();
+ String tierConfigsPrefix = String.format("%s.%s.", TIER_CONFIGS_PREFIX, tierBackend);
+ Map<String, Object> tierConfigs =
+ new HashMap<>(_instanceDataManagerConfiguration.subset(tierConfigsPrefix).toMap());
+ if (!tierConfigs.containsKey(READ_MODE)) {
+ tierConfigs.put(READ_MODE, getReadMode());
+ }
+ return new PinotConfiguration(tierConfigs);
+ }
+
+ @Override
public String toString() {
String configString = "";
configString += "Instance Id: " + getInstanceId();
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixServerStarter.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixServerStarter.java
index 0b2dfda..1a154e4 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixServerStarter.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixServerStarter.java
@@ -62,7 +62,7 @@ import org.apache.pinot.core.transport.TlsConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.core.util.TlsUtils;
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneIndexRefreshState;
-import org.apache.pinot.segment.local.segment.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.server.api.access.AccessControlFactory;
import org.apache.pinot.server.conf.ServerConf;
import org.apache.pinot.server.realtime.ControllerLeaderLocator;
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
index 252cd1d..dca2f9b 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
@@ -34,8 +34,8 @@ import org.apache.pinot.common.utils.fetcher.SegmentFetcherFactory;
import org.apache.pinot.core.data.manager.InstanceDataManager;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
import org.apache.pinot.segment.local.segment.index.loader.V3RemoveIndexException;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.crypt.PinotCrypter;
import org.apache.pinot.spi.crypt.PinotCrypterFactory;
import org.apache.pinot.spi.env.PinotConfiguration;
diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index d15c030..59dfe64 100644
--- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -27,10 +27,10 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.restlet.resources.TableSegments;
import org.apache.pinot.common.restlet.resources.TablesList;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.testng.Assert;
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
index eebcecd..7cb4730 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
@@ -54,4 +54,8 @@ public interface InstanceDataManagerConfig {
int getMaxParallelSegmentBuilds();
String getAuthToken();
+
+ String getTierBackend();
+
+ PinotConfiguration getTierConfigs();
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/SegmentDumpTool.java b/pinot-tools/src/main/java/org/apache/pinot/tools/SegmentDumpTool.java
index 9a8f60c..cc411d0 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/SegmentDumpTool.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/SegmentDumpTool.java
@@ -27,9 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
import org.apache.pinot.spi.data.Schema;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/ArrayBasedGlobalDictionaries.java b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/ArrayBasedGlobalDictionaries.java
index 9f84f50..6816b55 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/ArrayBasedGlobalDictionaries.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/ArrayBasedGlobalDictionaries.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.ByteArray;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/GlobalDictionaries.java b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/GlobalDictionaries.java
index 0a27dc7..762125b 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/GlobalDictionaries.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/GlobalDictionaries.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.tools.anonymizer;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
public interface GlobalDictionaries {
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/MapBasedGlobalDictionaries.java b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/MapBasedGlobalDictionaries.java
index a4b5781..3f6bc52 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/MapBasedGlobalDictionaries.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/MapBasedGlobalDictionaries.java
@@ -29,7 +29,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.ByteArray;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/PinotDataAndQueryAnonymizer.java b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/PinotDataAndQueryAnonymizer.java
index 00c988f..351eadb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/PinotDataAndQueryAnonymizer.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/anonymizer/PinotDataAndQueryAnonymizer.java
@@ -65,11 +65,11 @@ import org.apache.pinot.pql.parsers.pql2.ast.StarExpressionAstNode;
import org.apache.pinot.pql.parsers.pql2.ast.StringLiteralAstNode;
import org.apache.pinot.pql.parsers.pql2.ast.WhereAstNode;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkRunner.java
index 8f2796e..10d7369 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkRunner.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.tools.AbstractBaseCommand;
import org.apache.pinot.tools.Command;
import org.kohsuke.args4j.CmdLineParser;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Aggregation.java b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Aggregation.java
index 7a03eb0..ce68a01 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Aggregation.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Aggregation.java
@@ -24,8 +24,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.pinot.common.request.AggregationInfo;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.utils.Pair;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Projection.java b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Projection.java
index fba5968..65280ed 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Projection.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Projection.java
@@ -26,9 +26,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/SegmentQueryProcessor.java b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/SegmentQueryProcessor.java
index 40e2b10..2300054 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/SegmentQueryProcessor.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/SegmentQueryProcessor.java
@@ -33,10 +33,10 @@ import org.apache.pinot.common.request.GroupBy;
import org.apache.pinot.common.utils.request.FilterQueryTree;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.segment.local.segment.index.metadata.ColumnMetadata;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.segment.spi.index.metadata.ColumnMetadata;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Selection.java b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Selection.java
index 0e92327..a9951ca 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Selection.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/scan/query/Selection.java
@@ -21,8 +21,8 @@ package org.apache.pinot.tools.scan.query;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.utils.Pair;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
index 9eee56e..d3e53141 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
@@ -32,8 +32,8 @@ import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentColumnarIndexCreator;
-import org.apache.pinot.segment.local.segment.creator.impl.V1Constants;
import org.apache.pinot.segment.spi.IndexSegment;
+import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/PinotSegmentToAvroConverter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/PinotSegmentToAvroConverter.java
index 0df2b6d..f988f05 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/PinotSegmentToAvroConverter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/PinotSegmentToAvroConverter.java
@@ -27,8 +27,8 @@ import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData.Record;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.pinot.plugin.inputformat.avro.AvroUtils;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.data.readers.GenericRow;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
index 65dc642..ed62e40 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java
@@ -31,9 +31,9 @@ import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.core.minion.rollup.MergeRollupSegmentConverter;
import org.apache.pinot.core.minion.rollup.MergeType;
-import org.apache.pinot.segment.local.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.name.NormalizedDateSegmentNameGenerator;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.config.table.SegmentsValidationAndRetentionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org