You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2021/11/03 16:04:59 UTC
[pinot] branch master updated: Cleanup SegmentDirectoryLoader load
(#7682)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d8961b0 Cleanup SegmentDirectoryLoader load (#7682)
d8961b0 is described below
commit d8961b074ae2926bccce9c77542e977a13ffdecb
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Wed Nov 3 09:04:38 2021 -0700
Cleanup SegmentDirectoryLoader load (#7682)
* Cleanup SegmentDirectoryLoader load
* license header
---
.../pinot/core/minion/SegmentPurgerTest.java | 4 +-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 3 +-
.../immutable/ImmutableSegmentLoader.java | 23 ++++---
.../local/loader/LocalSegmentDirectoryLoader.java | 8 ++-
.../converter/SegmentV1V2ToV3FormatConverter.java | 5 +-
.../segment/index/loader/IndexLoadingConfig.java | 13 ++--
.../segment/index/creator/RawIndexCreatorTest.java | 4 +-
.../SegmentGenerationWithBytesTypeTest.java | 4 +-
.../local/segment/index/loader/LoaderTest.java | 19 ++++--
.../index/loader/SegmentPreProcessorTest.java | 70 ++++++++++++----------
.../segment/spi/loader/SegmentDirectoryLoader.java | 8 +--
...der.java => SegmentDirectoryLoaderContext.java} | 41 ++++++++-----
12 files changed, 122 insertions(+), 80 deletions(-)
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
index 2cf6702..96e6075 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/minion/SegmentPurgerTest.java
@@ -32,6 +32,7 @@ import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
@@ -163,7 +164,8 @@ public class SegmentPurgerTest {
Map<String, Object> props = new HashMap<>();
props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(purgedIndexDir.toURI(), new PinotConfiguration(props));
+ .load(purgedIndexDir.toURI(),
+ new SegmentDirectoryLoaderContext(_tableConfig, null, new PinotConfiguration(props)));
SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
assertTrue(reader.hasIndexFor(D1, ColumnIndexType.INVERTED_INDEX));
assertFalse(reader.hasIndexFor(D2, ColumnIndexType.INVERTED_INDEX));
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
index 7f34d13..4f7bf3d 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOfflineIndexReader.java
@@ -44,6 +44,7 @@ import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
@@ -121,7 +122,7 @@ public class BenchmarkOfflineIndexReader {
Map<String, Object> props = new HashMap<>();
props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(indexDir.toURI(), new PinotConfiguration(props));
+ .load(indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, new PinotConfiguration(props)));
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
// Forward index
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index a7b6d04..85f1158 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -39,6 +39,8 @@ import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
@@ -112,14 +114,14 @@ public class ImmutableSegmentLoader {
// Load the segment again for the configured tier backend. Default is 'local'.
PinotConfiguration tierConfigs = indexLoadingConfig.getTierConfigs();
- Map<String, Object> segDirConfigMap = tierConfigs.toMap();
- if (indexLoadingConfig.getTableNameWithType() != null) {
- segDirConfigMap.put(IndexLoadingConfig.TABLE_NAME_WITH_TYPE_KEY, indexLoadingConfig.getTableNameWithType());
- }
- PinotConfiguration segDirConfigs = new PinotConfiguration(segDirConfigMap);
- SegmentDirectory actualSegmentDirectory =
- SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getTierBackend())
- .load(indexDir.toURI(), segDirConfigs);
+ PinotConfiguration segDirConfigs = new PinotConfiguration(tierConfigs.toMap());
+ SegmentDirectoryLoaderContext segmentLoaderContext =
+ new SegmentDirectoryLoaderContext(indexLoadingConfig.getTableConfig(),
+ indexLoadingConfig.getInstanceId(), segDirConfigs);
+
+ SegmentDirectoryLoader segmentDirectoryLoader =
+ SegmentDirectoryLoaderRegistry.getSegmentDirectoryLoader(indexLoadingConfig.getTierBackend());
+ SegmentDirectory actualSegmentDirectory = segmentDirectoryLoader.load(indexDir.toURI(), segmentLoaderContext);
SegmentDirectory.Reader segmentReader = actualSegmentDirectory.createReader();
SegmentMetadataImpl segmentMetadata = actualSegmentDirectory.getSegmentMetadata();
@@ -199,8 +201,11 @@ public class ImmutableSegmentLoader {
throws Exception {
PinotConfiguration tierConfigs = indexLoadingConfig.getTierConfigs();
PinotConfiguration segDirConfigs = new PinotConfiguration(tierConfigs.toMap());
+ SegmentDirectoryLoaderContext segmentLoaderContext =
+ new SegmentDirectoryLoaderContext(indexLoadingConfig.getTableConfig(), indexLoadingConfig.getInstanceId(),
+ segDirConfigs);
SegmentDirectory segDir =
- SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader().load(indexDir.toURI(), segDirConfigs);
+ SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader().load(indexDir.toURI(), segmentLoaderContext);
try (SegmentPreProcessor preProcessor = new SegmentPreProcessor(segDir, indexLoadingConfig, schema)) {
preProcessor.process();
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java
index a3c7e77..b4c1ac0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/loader/LocalSegmentDirectoryLoader.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.net.URI;
import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentLoader;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -39,13 +40,14 @@ public class LocalSegmentDirectoryLoader implements SegmentDirectoryLoader {
/**
* Creates and loads the {@link SegmentLocalFSDirectory} which is the local implementation of {@link SegmentDirectory}
* @param indexDir segment index directory
- * @param segmentDirectoryConfig config containing values for instantiation of the SegmentDirectory
+ * @param segmentLoaderContext context for instantiation of the SegmentDirectory
* @return instance of {@link SegmentLocalFSDirectory}
*/
@Override
- public SegmentDirectory load(URI indexDir, PinotConfiguration segmentDirectoryConfig)
+ public SegmentDirectory load(URI indexDir, SegmentDirectoryLoaderContext segmentLoaderContext)
throws Exception {
+ PinotConfiguration segmentDirectoryConfigs = segmentLoaderContext.getSegmentDirectoryConfigs();
return new SegmentLocalFSDirectory(new File(indexDir),
- ReadMode.valueOf(segmentDirectoryConfig.getProperty(READ_MODE_KEY)));
+ ReadMode.valueOf(segmentDirectoryConfigs.getProperty(READ_MODE_KEY)));
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index ecb3fc3..cce906d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -38,6 +38,7 @@ import org.apache.pinot.segment.spi.converter.SegmentFormatConverter;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
@@ -141,9 +142,9 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
PinotConfiguration configuration = new PinotConfiguration(props);
try (SegmentDirectory v2Segment = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(v2Directory.toURI(), configuration);
+ .load(v2Directory.toURI(), new SegmentDirectoryLoaderContext(null, null, configuration));
SegmentDirectory v3Segment = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(v3Directory.toURI(), configuration)) {
+ .load(v3Directory.toURI(), new SegmentDirectoryLoaderContext(null, null, configuration))) {
// for each dictionary and each fwdIndex, copy that to newDirectory buffer
Set<String> allColumns = v2Metadata.getAllColumns();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
index c91602a..1cb7e0c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -49,7 +49,6 @@ import org.apache.pinot.spi.utils.ReadMode;
public class IndexLoadingConfig {
private static final int DEFAULT_REALTIME_AVG_MULTI_VALUE_COUNT = 2;
public static final String DEFAULT_TIER_BACKEND = "local";
- public static final String TABLE_NAME_WITH_TYPE_KEY = "tableNameWithType";
private String _tableNameWithType;
private ReadMode _readMode = ReadMode.DEFAULT_MODE;
@@ -86,6 +85,8 @@ public class IndexLoadingConfig {
private String _tierBackend;
private PinotConfiguration _tierConfigs;
+ private String _instanceId;
+
public IndexLoadingConfig(InstanceDataManagerConfig instanceDataManagerConfig, TableConfig tableConfig) {
extractFromInstanceConfig(instanceDataManagerConfig);
extractFromTableConfig(tableConfig);
@@ -227,6 +228,8 @@ public class IndexLoadingConfig {
if (instanceDataManagerConfig == null) {
return;
}
+ _instanceId = instanceDataManagerConfig.getInstanceId();
+
ReadMode instanceReadMode = instanceDataManagerConfig.getReadMode();
if (instanceReadMode != null) {
_readMode = instanceReadMode;
@@ -259,10 +262,6 @@ public class IndexLoadingConfig {
public IndexLoadingConfig() {
}
- public String getTableNameWithType() {
- return _tableNameWithType;
- }
-
public ReadMode getReadMode() {
return _readMode;
}
@@ -475,4 +474,8 @@ public class IndexLoadingConfig {
}
return _tierConfigs;
}
+
+ public String getInstanceId() {
+ return _instanceId;
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
index d8cafdc..f1c7e5a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/RawIndexCreatorTest.java
@@ -37,6 +37,7 @@ import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunk
import org.apache.pinot.segment.local.segment.index.readers.forward.VarByteChunkSVForwardIndexReader;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
@@ -328,7 +329,8 @@ public class RawIndexCreatorTest {
Map<String, Object> props = new HashMap<>();
props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
_segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(driver.getOutputDirectory().toURI(), new PinotConfiguration(props));
+ .load(driver.getOutputDirectory().toURI(),
+ new SegmentDirectoryLoaderContext(tableConfig, null, new PinotConfiguration(props)));
_segmentReader = _segmentDirectory.createReader();
recordReader.rewind();
return recordReader;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
index 9015c40..fbd0866 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithBytesTypeTest.java
@@ -45,6 +45,7 @@ import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
@@ -272,7 +273,8 @@ public class SegmentGenerationWithBytesTypeTest {
Map<String, Object> props = new HashMap<>();
props.put(LocalSegmentDirectoryLoader.READ_MODE_KEY, ReadMode.mmap.toString());
SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(driver.getOutputDirectory().toURI(), new PinotConfiguration(props));
+ .load(driver.getOutputDirectory().toURI(),
+ new SegmentDirectoryLoaderContext(_tableConfig, null, new PinotConfiguration(props)));
recordReader.rewind();
return recordReader;
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index 7d7871c..db37bab 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -41,6 +41,7 @@ import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoader;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
@@ -198,7 +199,8 @@ public class LoaderTest {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.LEGACY_STRING_PAD_CHAR);
- SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
+ SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
SegmentDirectory.Reader reader = segmentDir.createReader();
PinotDataBuffer dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
StringDictionary dict =
@@ -219,7 +221,8 @@ public class LoaderTest {
segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.LEGACY_STRING_PAD_CHAR);
- segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
+ segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
reader = segmentDir.createReader();
dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
dict = new StringDictionary(dictionaryBuffer, columnMetadata.getCardinality(), columnMetadata.getColumnMaxLength(),
@@ -239,7 +242,8 @@ public class LoaderTest {
segmentMetadata = new SegmentMetadataImpl(segmentDirectory);
columnMetadata = segmentMetadata.getColumnMetadataFor("name");
Assert.assertEquals(columnMetadata.getPaddingCharacter(), V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
- segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(), _pinotConfiguration);
+ segmentDir = _localSegmentDirectoryLoader.load(segmentDirectory.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
reader = segmentDir.createReader();
dictionaryBuffer = reader.getIndexFor("name", ColumnIndexType.DICTIONARY);
dict = new StringDictionary(dictionaryBuffer, columnMetadata.getCardinality(), columnMetadata.getColumnMaxLength(),
@@ -324,7 +328,8 @@ public class LoaderTest {
// check that index dir is not in V1 format (the only subdir it should have is V3)
verifyIndexDirIsV3(_indexDir);
- SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
+ SegmentDirectory segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
SegmentDirectory.Reader reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
@@ -340,7 +345,8 @@ public class LoaderTest {
Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(_indexDir, SegmentVersion.v3).exists());
// check that index dir is not in V1 format (the only subdir it should have is V3)
verifyIndexDirIsV3(_indexDir);
- segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
+ segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
@@ -414,7 +420,8 @@ public class LoaderTest {
verifyIndexDirIsV3(_indexDir);
fstIndexFile = SegmentDirectoryPaths.findFSTIndexIndexFile(_indexDir, FST_INDEX_COL_NAME);
Assert.assertNull(fstIndexFile);
- segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(), _pinotConfiguration);
+ segmentDir = _localSegmentDirectoryLoader.load(_indexDir.toURI(),
+ new SegmentDirectoryLoaderContext(null, null, _pinotConfiguration));
reader = segmentDir.createReader();
Assert.assertNotNull(reader);
Assert.assertTrue(reader.hasIndexFor(FST_INDEX_COL_NAME, ColumnIndexType.FST_INDEX));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index 8a2fb82..2b111a4 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -44,6 +44,7 @@ import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
import org.apache.pinot.segment.spi.creator.SegmentVersion;
import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderContext;
import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
@@ -249,14 +250,16 @@ public class SegmentPreProcessorTest {
_indexLoadingConfig.getNoDictionaryColumns().add(EXISTING_STRING_COL_RAW);
constructV3Segment();
SegmentDirectory segmentDirectory =
- SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader().load(_indexDir.toURI(), _configuration);
+ SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor v3Processor =
new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchemaWithFST);
expectThrows(UnsupportedOperationException.class, () -> v3Processor.process());
constructV1Segment();
segmentDirectory =
- SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader().load(_indexDir.toURI(), _configuration);
+ SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor v1Processor =
new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, _newColumnsSchemaWithFST);
expectThrows(UnsupportedOperationException.class, () -> v1Processor.process());
@@ -419,7 +422,7 @@ public class SegmentPreProcessorTest {
throws Exception {
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, schema)) {
processor.process();
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir);
@@ -436,7 +439,8 @@ public class SegmentPreProcessorTest {
assertEquals(columnMetadata.isAutoGenerated(), isAutoGenerated);
try (SegmentDirectory segmentDirectory1 = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory1.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory1.createReader()) {
assertTrue(reader.hasIndexFor(column, indexType));
assertTrue(reader.hasIndexFor(column, ColumnIndexType.FORWARD_INDEX));
// if the text index is enabled on a new column with dictionary,
@@ -522,7 +526,8 @@ public class SegmentPreProcessorTest {
checkInvertedIndexCreation(false);
long addedLength = 0L;
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
// 8 bytes overhead is for checking integrity of the segment.
addedLength += reader.getIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX).size() + 8;
addedLength += reader.getIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX).size() + 8;
@@ -544,7 +549,8 @@ public class SegmentPreProcessorTest {
private void checkInvertedIndexCreation(boolean reCreate)
throws Exception {
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
if (reCreate) {
assertTrue(reader.hasIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX));
assertTrue(reader.hasIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX));
@@ -559,13 +565,14 @@ public class SegmentPreProcessorTest {
}
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
assertTrue(reader.hasIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX));
assertTrue(reader.hasIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX));
assertTrue(reader.hasIndexFor(COLUMN7_NAME, ColumnIndexType.INVERTED_INDEX));
@@ -590,7 +597,7 @@ public class SegmentPreProcessorTest {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_indexDir);
assertNotNull(segmentMetadata.getColumnMetadataFor(NEW_INT_SV_DIMENSION_COLUMN_NAME));
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig,
_newColumnsSchema3)) {
processor.process();
@@ -634,7 +641,7 @@ public class SegmentPreProcessorTest {
segmentMetadata = new SegmentMetadataImpl(_indexDir);
assertNotNull(segmentMetadata.getColumnMetadataFor(NEW_INT_SV_DIMENSION_COLUMN_NAME));
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig,
_newColumnsSchema3)) {
processor.process();
@@ -662,7 +669,7 @@ public class SegmentPreProcessorTest {
throws Exception {
// Update default value.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig,
_newColumnsSchema1)) {
processor.process();
@@ -730,7 +737,8 @@ public class SegmentPreProcessorTest {
// Check dictionary and forward index exist.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
assertTrue(reader.hasIndexFor(NEW_INT_METRIC_COLUMN_NAME, ColumnIndexType.DICTIONARY));
assertTrue(reader.hasIndexFor(NEW_INT_METRIC_COLUMN_NAME, ColumnIndexType.FORWARD_INDEX));
assertTrue(reader.hasIndexFor(NEW_LONG_METRIC_COLUMN_NAME, ColumnIndexType.DICTIONARY));
@@ -751,7 +759,7 @@ public class SegmentPreProcessorTest {
// For the second schema, we changed the default value for column 'newIntMetric' to 2, and added default value
// 'abcd' (keep the same length as 'null') to column 'newStringMVDimension'.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig,
_newColumnsSchema2)) {
processor.process();
@@ -790,7 +798,7 @@ public class SegmentPreProcessorTest {
IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NONE);
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
@@ -807,7 +815,7 @@ public class SegmentPreProcessorTest {
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.TIME);
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
@@ -824,7 +832,7 @@ public class SegmentPreProcessorTest {
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NON_METRIC);
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
@@ -841,7 +849,7 @@ public class SegmentPreProcessorTest {
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.ALL);
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, null)) {
processor.process();
}
@@ -890,7 +898,7 @@ public class SegmentPreProcessorTest {
// Create all kinds of indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
@@ -902,7 +910,7 @@ public class SegmentPreProcessorTest {
// Remove all kinds of indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(), null)) {
processor.process();
}
@@ -927,7 +935,7 @@ public class SegmentPreProcessorTest {
// There are a few indices initially. Remove them to prepare an initial state.
long initFileSize = singleFileIndex.length();
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(), null)) {
processor.process();
}
@@ -946,14 +954,15 @@ public class SegmentPreProcessorTest {
// Create all kinds of indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
long addedLength = 0;
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
addedLength += reader.getIndexFor(strColumn, ColumnIndexType.INVERTED_INDEX).size() + 8;
addedLength += reader.getIndexFor(strColumn, ColumnIndexType.RANGE_INDEX).size() + 8;
addedLength += reader.getIndexFor(strColumn, ColumnIndexType.FST_INDEX).size() + 8;
@@ -964,7 +973,7 @@ public class SegmentPreProcessorTest {
// Remove all kinds of indices, and size gets back initial size.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(), null)) {
processor.process();
}
@@ -978,7 +987,7 @@ public class SegmentPreProcessorTest {
// Remove all indices and add the two derived columns for H3 and Json index.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(),
_newColumnsSchemaWithH3Json)) {
processor.process();
@@ -1002,7 +1011,7 @@ public class SegmentPreProcessorTest {
// Create H3 and Json indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
@@ -1011,7 +1020,7 @@ public class SegmentPreProcessorTest {
// Remove H3 and Json indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(), null)) {
processor.process();
}
@@ -1033,7 +1042,7 @@ public class SegmentPreProcessorTest {
// There are a few indices initially. Remove them to prepare an initial state.
// Also use the schema with columns for H3 and Json index to add those columns.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(),
_newColumnsSchemaWithH3Json)) {
processor.process();
@@ -1050,14 +1059,15 @@ public class SegmentPreProcessorTest {
// Create H3 and Json indices.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, _indexLoadingConfig, null)) {
processor.process();
}
long addedLength = 0;
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration); SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
addedLength += reader.getIndexFor("newH3Col", ColumnIndexType.H3_INDEX).size() + 8;
addedLength += reader.getIndexFor("newJsonCol", ColumnIndexType.JSON_INDEX).size() + 8;
}
@@ -1065,7 +1075,7 @@ public class SegmentPreProcessorTest {
// Remove H3 and Json indices, and size gets back to initial.
try (SegmentDirectory segmentDirectory = SegmentDirectoryLoaderRegistry.getLocalSegmentDirectoryLoader()
- .load(_indexDir.toURI(), _configuration);
+ .load(_indexDir.toURI(), new SegmentDirectoryLoaderContext(null, null, _configuration));
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, new IndexLoadingConfig(), null)) {
processor.process();
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
index 337ae81..d4e60a8 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
@@ -20,7 +20,6 @@ package org.apache.pinot.segment.spi.loader;
import java.net.URI;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.env.PinotConfiguration;
/**
@@ -31,11 +30,8 @@ public interface SegmentDirectoryLoader {
/**
* Creates the {@link SegmentDirectory} instance
* @param indexDir index directory
- * @param segmentDirectoryConfig config for SegmentDirectory, containing all properties needed to instantiate the
- * {@link SegmentDirectory}
- * e.g. readMode (MMAP/HEAP) or
- * properties specific to the tier backend (deep store access configs)
+ * @param segmentDirectoryLoaderContext context for loading this segment
*/
- SegmentDirectory load(URI indexDir, PinotConfiguration segmentDirectoryConfig)
+ SegmentDirectory load(URI indexDir, SegmentDirectoryLoaderContext segmentDirectoryLoaderContext)
throws Exception;
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
similarity index 54%
copy from pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
copy to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
index 337ae81..9aecce9 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoader.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/loader/SegmentDirectoryLoaderContext.java
@@ -18,24 +18,35 @@
*/
package org.apache.pinot.segment.spi.loader;
-import java.net.URI;
-import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
/**
- * Interface for creating and loading the {@link SegmentDirectory} instance using provided config
+ * Context for {@link SegmentDirectoryLoader}
*/
-public interface SegmentDirectoryLoader {
-
- /**
- * Creates the {@link SegmentDirectory} instance
- * @param indexDir index directory
- * @param segmentDirectoryConfig config for SegmentDirectory, containing all properties needed to instantiate the
- * {@link SegmentDirectory}
- * e.g. readMode (MMAP/HEAP) or
- * properties specific to the tier backend (deep store access configs)
- */
- SegmentDirectory load(URI indexDir, PinotConfiguration segmentDirectoryConfig)
- throws Exception;
+public class SegmentDirectoryLoaderContext {
+
+ private final TableConfig _tableConfig;
+ private final String _instanceId;
+ private final PinotConfiguration _segmentDirectoryConfigs;
+
+ public SegmentDirectoryLoaderContext(TableConfig tableConfig, String instanceId,
+ PinotConfiguration segmentDirectoryConfigs) {
+ _tableConfig = tableConfig;
+ _instanceId = instanceId;
+ _segmentDirectoryConfigs = segmentDirectoryConfigs;
+ }
+
+ public TableConfig getTableConfig() {
+ return _tableConfig;
+ }
+
+ public String getInstanceId() {
+ return _instanceId;
+ }
+
+ public PinotConfiguration getSegmentDirectoryConfigs() {
+ return _segmentDirectoryConfigs;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org