You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/11/07 08:33:10 UTC
(pinot) branch master updated (d177866527 -> 06b58d4dcd)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
from d177866527 [bugfix][multistage] explicit warning flags set on each stage stats (#11936)
new 8f69299aef upgrade lucene to 9.8.0
new 00cf7a8295 Try lucene-backward-codecs package
new 06b58d4dcd adding lucene 80 compatible test
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
compatibility-verifier/compCheck.sh | 6 +++++
pinot-core/pom.xml | 12 ++++++---
pinot-segment-local/pom.xml | 13 +++++++---
.../impl/inv/text/LuceneFSTIndexCreator.java | 4 +--
.../creator/impl/text/LuceneTextIndexCreator.java | 2 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 4 +--
.../local/segment/index/fst/FstIndexType.java | 5 ++--
.../loader/invertedindex/FSTIndexHandler.java | 4 +--
.../index/readers/LuceneFSTIndexReader.java | 3 ++-
.../index/readers/text/LuceneTextIndexReader.java | 18 ++++++++------
.../local/segment/index/text/TextIndexType.java | 3 ++-
.../local/segment/store/TextIndexUtils.java | 11 ++++++---
.../pinot/segment/local/utils/fst/FSTBuilder.java | 19 +++++++--------
.../segment/local/utils/fst/RegexpMatcher.java | 8 +++---
.../utils/nativefst/NativeFSTIndexCreator.java | 2 +-
.../index/creator/LuceneFSTIndexCreatorTest.java | 4 +--
.../index/creator/NativeFSTIndexCreatorTest.java | 4 +--
.../local/segment/index/loader/LoaderTest.java | 22 ++++++++---------
.../index/loader/SegmentPreProcessorTest.java | 4 +--
.../text/LuceneTextIndexCompatibleTest.java | 18 +++++++-------
.../segment/store/FilePerIndexDirectoryTest.java | 7 +++---
.../store/SingleFileIndexDirectoryTest.java | 7 +++---
.../segment/local/utils/fst/FSTBuilderTest.java | 4 +--
.../data/lucene_80_index/Text.lucene.index/_0.cfe | Bin 0 -> 299 bytes
.../data/lucene_80_index/Text.lucene.index/_0.cfs | Bin 0 -> 136123 bytes
.../data/lucene_80_index/Text.lucene.index/_0.si | Bin 0 -> 370 bytes
.../lucene_80_index/Text.lucene.index/segments_1 | Bin 0 -> 137 bytes
.../lucene_80_index/Text.lucene.index/write.lock | 0
.../org/apache/pinot/segment/spi/V1Constants.java | 4 ++-
.../segment/spi/store/SegmentDirectoryPaths.java | 18 +++++++++++---
.../fineFoodReviews_offline_table_config.json | 12 +++++++++
pom.xml | 27 ++++++++++++++++++++-
32 files changed, 160 insertions(+), 85 deletions(-)
copy pinot-core/src/test/java/org/apache/pinot/core/metadata/MetadataExtractorFactoryTest.java => pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java (63%)
create mode 100644 pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe
create mode 100644 pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs
create mode 100644 pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si
create mode 100644 pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1
copy .trivyignore => pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock (100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
(pinot) 01/03: upgrade lucene to 9.8.0
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 8f69299aefcd41b834e8671ebc55b0eaf4ea2834
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Fri Oct 27 14:10:38 2023 -0700
upgrade lucene to 9.8.0
---
pinot-core/pom.xml | 2 +-
pinot-segment-local/pom.xml | 2 +-
.../creator/impl/inv/text/LuceneFSTIndexCreator.java | 2 +-
.../segment/index/readers/LuceneFSTIndexReader.java | 3 ++-
.../pinot/segment/local/utils/fst/FSTBuilder.java | 19 +++++++++----------
.../pinot/segment/local/utils/fst/RegexpMatcher.java | 8 ++++----
.../pinot/segment/local/utils/fst/FSTBuilderTest.java | 4 ++--
.../org/apache/pinot/segment/spi/V1Constants.java | 6 +++---
pom.xml | 2 +-
9 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index f47444e42e..89ced53e82 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -258,7 +258,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
+ <artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
</dependencies>
diff --git a/pinot-segment-local/pom.xml b/pinot-segment-local/pom.xml
index 9724e3bfd0..2d90fbed46 100644
--- a/pinot-segment-local/pom.xml
+++ b/pinot-segment-local/pom.xml
@@ -69,7 +69,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
+ <artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
index ab9d7490e8..695ce4b1ea 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
@@ -97,7 +97,7 @@ public class LuceneFSTIndexCreator implements FSTIndexCreator {
fileOutputStream = new FileOutputStream(_fstIndexFile);
FST<Long> fst = _fstBuilder.done();
OutputStreamDataOutput d = new OutputStreamDataOutput(fileOutputStream);
- fst.save(d);
+ fst.save(d, d);
} finally {
if (fileOutputStream != null) {
fileOutputStream.close();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
index f859889e7f..6e579562c4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/LuceneFSTIndexReader.java
@@ -51,7 +51,8 @@ public class LuceneFSTIndexReader implements TextIndexReader {
_dataBuffer = pinotDataBuffer;
_dataBufferIndexInput = new PinotBufferIndexInput(_dataBuffer, 0L, _dataBuffer.size());
- _readFST = new FST(_dataBufferIndexInput, PositiveIntOutputs.getSingleton(), new OffHeapFSTStore());
+ _readFST =
+ new FST(_dataBufferIndexInput, _dataBufferIndexInput, PositiveIntOutputs.getSingleton(), new OffHeapFSTStore());
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/FSTBuilder.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/FSTBuilder.java
index c998b7113d..0a4596d173 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/FSTBuilder.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/FSTBuilder.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.util.Map;
import java.util.SortedMap;
import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
import org.slf4j.Logger;
@@ -36,20 +36,19 @@ import org.slf4j.LoggerFactory;
*/
public class FSTBuilder {
public static final Logger LOGGER = LoggerFactory.getLogger(FSTBuilder.class);
- private Builder<Long> _builder = new Builder<>(FST.INPUT_TYPE.BYTE4, PositiveIntOutputs.getSingleton());
- private IntsRefBuilder _scratch = new IntsRefBuilder();
+ private final FSTCompiler<Long> _builder = new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, PositiveIntOutputs.getSingleton());
+ private final IntsRefBuilder _scratch = new IntsRefBuilder();
- public static FST buildFST(SortedMap<String, Integer> input)
+ public static FST<Long> buildFST(SortedMap<String, Integer> input)
throws IOException {
PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton();
- Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, fstOutput);
+ FSTCompiler<Long> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, fstOutput);
IntsRefBuilder scratch = new IntsRefBuilder();
for (Map.Entry<String, Integer> entry : input.entrySet()) {
- builder.add(Util.toUTF16(entry.getKey(), scratch), entry.getValue().longValue());
+ fstCompiler.add(Util.toUTF16(entry.getKey(), scratch), entry.getValue().longValue());
}
- FST<Long> result = builder.finish();
- return result;
+ return fstCompiler.compile();
}
public void addEntry(String key, Integer value)
@@ -57,8 +56,8 @@ public class FSTBuilder {
_builder.add(Util.toUTF16(key, _scratch), value.longValue());
}
- public FST done()
+ public FST<Long> done()
throws IOException {
- return _builder.finish();
+ return _builder.compile();
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/RegexpMatcher.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/RegexpMatcher.java
index eb6e0d9866..f761c2b8dd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/RegexpMatcher.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/fst/RegexpMatcher.java
@@ -122,16 +122,16 @@ public class RegexpMatcher {
newInput.copyInts(currentInput.get());
newInput.append(t.min);
queue.add(new Path<Long>(t.dest, new FST.Arc<Long>().copyFrom(nextArc),
- _fst.outputs.add(path._output, nextArc.output), newInput));
+ _fst.outputs.add(path._output, nextArc.output()), newInput));
}
} else {
FST.Arc<Long> nextArc = Util.readCeilArc(min, _fst, path._fstNode, scratchArc, fstReader);
- while (nextArc != null && nextArc.label <= max) {
+ while (nextArc != null && nextArc.label() <= max) {
final IntsRefBuilder newInput = new IntsRefBuilder();
newInput.copyInts(currentInput.get());
- newInput.append(nextArc.label);
+ newInput.append(nextArc.label());
queue.add(new Path<>(t.dest, new FST.Arc<Long>().copyFrom(nextArc),
- _fst.outputs.add(path._output, nextArc.output), newInput));
+ _fst.outputs.add(path._output, nextArc.output()), newInput));
nextArc = nextArc.isLast() ? null : _fst.readNextRealArc(nextArc, fstReader);
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
index 883faeb447..493e7b3449 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/fst/FSTBuilderTest.java
@@ -69,7 +69,7 @@ public class FSTBuilderTest {
File outputFile = new File(TEMP_DIR, "test.lucene");
FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
OutputStreamDataOutput d = new OutputStreamDataOutput(fileOutputStream);
- fst.save(d);
+ fst.save(d, d);
fileOutputStream.close();
Outputs<Long> outputs = PositiveIntOutputs.getSingleton();
@@ -78,7 +78,7 @@ public class FSTBuilderTest {
PinotDataBuffer pinotDataBuffer =
PinotDataBuffer.mapFile(fstFile, true, 0, fstFile.length(), ByteOrder.BIG_ENDIAN, "");
PinotBufferIndexInput indexInput = new PinotBufferIndexInput(pinotDataBuffer, 0L, fstFile.length());
- FST<Long> readFST = new FST(indexInput, outputs, new OffHeapFSTStore());
+ FST<Long> readFST = new FST(indexInput, indexInput, outputs, new OffHeapFSTStore());
List<Long> results = RegexpMatcher.regexMatch("hello.*123", fst);
Assert.assertEquals(results.size(), 1);
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 9f2c02fddd..8900510ab6 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -44,14 +44,14 @@ public class V1Constants {
public static final String UNSORTED_MV_FORWARD_INDEX_FILE_EXTENSION = ".mv.fwd";
public static final String BITMAP_INVERTED_INDEX_FILE_EXTENSION = ".bitmap.inv";
public static final String BITMAP_RANGE_INDEX_FILE_EXTENSION = ".bitmap.range";
- public static final String FST_INDEX_FILE_EXTENSION = ".lucene.fst";
+ public static final String FST_INDEX_FILE_EXTENSION = ".lucene.v9.fst";
public static final String JSON_INDEX_FILE_EXTENSION = ".json.idx";
public static final String NATIVE_TEXT_INDEX_FILE_EXTENSION = ".nativetext.idx";
public static final String H3_INDEX_FILE_EXTENSION = ".h3.idx";
public static final String BLOOM_FILTER_FILE_EXTENSION = ".bloom";
public static final String NULLVALUE_VECTOR_FILE_EXTENSION = ".bitmap.nullvalue";
- public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
- public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
+ public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.v9.mapping";
+ public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.v9.index";
}
public static class MetadataKeys {
diff --git a/pom.xml b/pom.xml
index 7c9ffcb4b3..29a739be48 100644
--- a/pom.xml
+++ b/pom.xml
@@ -138,7 +138,7 @@
<jsonsmart.version>2.5.0</jsonsmart.version>
<quartz.version>2.3.2</quartz.version>
<calcite.version>1.30.0</calcite.version>
- <lucene.version>8.2.0</lucene.version>
+ <lucene.version>9.8.0</lucene.version>
<reflections.version>0.9.11</reflections.version>
<!-- commons-configuration, hadoop-common, hadoop-client use commons-lang -->
<commons-lang.version>2.6</commons-lang.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
(pinot) 03/03: adding lucene 80 compatible test
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 06b58d4dcd723923f7fb61593115b6b44114edd0
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Thu Nov 2 20:03:21 2023 -0700
adding lucene 80 compatible test
---
.../converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../index/readers/text/LuceneTextIndexReader.java | 2 +-
.../local/segment/store/TextIndexUtils.java | 2 +-
.../local/segment/index/loader/LoaderTest.java | 10 +++---
.../text/LuceneTextIndexCompatibleTest.java | 36 +++++++++++++++++++++
.../segment/store/FilePerIndexDirectoryTest.java | 4 +--
.../store/SingleFileIndexDirectoryTest.java | 4 +--
.../data/lucene_80_index/Text.lucene.index/_0.cfe | Bin 0 -> 299 bytes
.../data/lucene_80_index/Text.lucene.index/_0.cfs | Bin 0 -> 136123 bytes
.../data/lucene_80_index/Text.lucene.index/_0.si | Bin 0 -> 370 bytes
.../lucene_80_index/Text.lucene.index/segments_1 | Bin 0 -> 137 bytes
.../lucene_80_index/Text.lucene.index/write.lock | 0
.../org/apache/pinot/segment/spi/V1Constants.java | 1 -
.../segment/spi/store/SegmentDirectoryPaths.java | 9 ++----
pom.xml | 1 +
15 files changed, 51 insertions(+), 20 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index d6cc8b1515..a65b58fd32 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -243,7 +243,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
// to moving the lucene text index files, we need to move the
// docID mapping/cache file created by us in v1/v2 during an earlier
// load of the segment.
- String docIDFileSuffix = V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ String docIDFileSuffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
File[] textIndexDocIdMappingFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
index 0d4646e1da..e249412e6b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
@@ -190,7 +190,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
throws Exception {
int length = Integer.BYTES * numDocs;
File docIdMappingFile = new File(SegmentDirectoryPaths.findSegmentDirectory(segmentIndexDir),
- column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
// The mapping is local to a segment. It is created on the server during segment load.
// Unless we are running Pinot on Solaris/SPARC, the underlying architecture is
// LITTLE_ENDIAN (Linux/x86). So use that as byte order.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
index 2d0510c486..235025acde 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
@@ -49,7 +49,7 @@ public class TextIndexUtils {
FileUtils.deleteQuietly(luceneMappingFile);
File luceneV9IndexFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
FileUtils.deleteQuietly(luceneV9IndexFile);
- File luceneV9MappingFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ File luceneV9MappingFile = new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
FileUtils.deleteQuietly(luceneV9MappingFile);
// Remove the native index file
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index bc64d539b0..6c73f20e17 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -608,7 +608,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -636,7 +636,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -686,7 +686,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -712,7 +712,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -738,7 +738,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java
new file mode 100644
index 0000000000..5ac2c6a17a
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers.text;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class LuceneTextIndexCompatibleTest {
+
+ @Test
+ public void testLucene80IndexReader() {
+ File indexPath =
+ new File(LuceneTextIndexCompatibleTest.class.getClassLoader().getResource("data/lucene_80_index").getPath());
+ LuceneTextIndexReader lucene80Index = new LuceneTextIndexReader("Text", indexPath, 1000, ImmutableMap.of());
+ Assert.assertNotNull(lucene80Index);
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index e115414b3e..ca933e01b3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -232,11 +232,11 @@ public class FilePerIndexDirectoryTest {
fpi.removeIndex("foo", StandardIndexes.text());
assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertTrue(
- new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 79778c8104..3c8ba99688 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -265,11 +265,11 @@ public class SingleFileIndexDirectoryTest {
sfd.removeIndex("foo", StandardIndexes.text());
assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertTrue(
- new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe
new file mode 100644
index 0000000000..6d4909e1bd
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs
new file mode 100644
index 0000000000..f57db05798
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si
new file mode 100644
index 0000000000..e92713aad5
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1 b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1
new file mode 100644
index 0000000000..d67a947f7a
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1 differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 218d174614..887bd87ff5 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -53,7 +53,6 @@ public class V1Constants {
public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
public static final String LUCENE_V9_FST_INDEX_FILE_EXTENSION = ".lucene.v9.fst";
- public static final String LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.v9.mapping";
public static final String LUCENE_V9_TEXT_INDEX_FILE_EXTENSION = ".lucene.v9.index";
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
index 98b60df91e..2f1cf67b81 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
@@ -113,13 +113,8 @@ public class SegmentDirectoryPaths {
@Nullable
@VisibleForTesting
public static File findTextIndexDocIdMappingFile(File indexDir, String column) {
- String file = column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
- File formatFile = findFormatFile(indexDir, file);
- if (formatFile == null) {
- file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
- formatFile = findFormatFile(indexDir, file);
- }
- return formatFile;
+ String file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ return findFormatFile(indexDir, file);
}
/**
diff --git a/pom.xml b/pom.xml
index fac14ba080..7f236c1ae7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2028,6 +2028,7 @@
<exclude>**/*.ser</exclude>
<exclude>**/*.v1</exclude>
<exclude>**/*.v2</exclude>
+ <exclude>**/Text.lucene.index/**</exclude>
<exclude>**/appAssemblerScriptTemplate</exclude>
<exclude>doap_Pinot.rdf</exclude>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
(pinot) 02/03: Try lucene-backward-codecs package
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 00cf7a829540814e4623e77f5cc1f4540520e411
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Tue Oct 31 15:09:15 2023 -0700
Try lucene-backward-codecs package
---
compatibility-verifier/compCheck.sh | 6 ++++
pinot-core/pom.xml | 10 +++++--
pinot-segment-local/pom.xml | 11 ++++++--
.../impl/inv/text/LuceneFSTIndexCreator.java | 2 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 2 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 6 ++--
.../local/segment/index/fst/FstIndexType.java | 5 ++--
.../loader/invertedindex/FSTIndexHandler.java | 4 +--
.../index/readers/text/LuceneTextIndexReader.java | 20 ++++++++------
.../local/segment/index/text/TextIndexType.java | 3 +-
.../local/segment/store/TextIndexUtils.java | 11 ++++++--
.../utils/nativefst/NativeFSTIndexCreator.java | 2 +-
.../index/creator/LuceneFSTIndexCreatorTest.java | 4 +--
.../index/creator/NativeFSTIndexCreatorTest.java | 4 +--
.../local/segment/index/loader/LoaderTest.java | 32 +++++++++++-----------
.../index/loader/SegmentPreProcessorTest.java | 4 +--
.../segment/store/FilePerIndexDirectoryTest.java | 9 +++---
.../store/SingleFileIndexDirectoryTest.java | 9 +++---
.../org/apache/pinot/segment/spi/V1Constants.java | 9 ++++--
.../segment/spi/store/SegmentDirectoryPaths.java | 27 ++++++++++++++----
.../fineFoodReviews_offline_table_config.json | 12 ++++++++
pom.xml | 24 ++++++++++++++++
22 files changed, 148 insertions(+), 68 deletions(-)
diff --git a/compatibility-verifier/compCheck.sh b/compatibility-verifier/compCheck.sh
index 0803a06b9c..8ed924c6c8 100755
--- a/compatibility-verifier/compCheck.sh
+++ b/compatibility-verifier/compCheck.sh
@@ -219,6 +219,12 @@ function stopServices() {
stopService server
stopService zookeeper
stopService kafka
+ echo "Controller logs:"
+ cat ${LOG_DIR}/controller.*.log
+ echo "Broker logs:"
+ cat ${LOG_DIR}/broker.*.log
+ echo "Server logs:"
+ cat ${LOG_DIR}/server.*.log
echo "Cluster stopped."
}
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index 89ced53e82..815bf75291 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -246,21 +246,25 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
+ <!-- Lucene dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
- <version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
- <version>${lucene.version}</version>
</dependency>
+ <!-- Lucene dependencies end -->
</dependencies>
<profiles>
<profile>
diff --git a/pinot-segment-local/pom.xml b/pinot-segment-local/pom.xml
index 2d90fbed46..5b2e5e24c7 100644
--- a/pinot-segment-local/pom.xml
+++ b/pinot-segment-local/pom.xml
@@ -57,21 +57,26 @@
<groupId>com.uber</groupId>
<artifactId>h3</artifactId>
</dependency>
+
+ <!-- Lucene dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
- <version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
- <version>${lucene.version}</version>
</dependency>
+ <!-- Lucene dependencies end -->
+
<dependency>
<groupId>org.apache.datasketches</groupId>
<artifactId>datasketches-java</artifactId>
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
index 695ce4b1ea..335449fe96 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
@@ -56,7 +56,7 @@ public class LuceneFSTIndexCreator implements FSTIndexCreator {
*/
public LuceneFSTIndexCreator(File indexDir, String columnName, String[] sortedEntries)
throws IOException {
- _fstIndexFile = new File(indexDir, columnName + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
+ _fstIndexFile = new File(indexDir, columnName + V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION);
_fstBuilder = new FSTBuilder();
_dictId = 0;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 3d96f9b226..771b42da3f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -196,7 +196,7 @@ public class LuceneTextIndexCreator extends AbstractTextIndexCreator {
}
private File getV1TextIndexFile(File indexDir) {
- String luceneIndexDirectory = _textColumn + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String luceneIndexDirectory = _textColumn + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION;
return new File(indexDir, luceneIndexDirectory);
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index f8c6819efb..d6cc8b1515 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -110,7 +110,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
if (file.isFile() && file.exists()) {
FileUtils.deleteQuietly(file);
}
- if (file.isDirectory() && file.getName().endsWith(V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION)) {
+ if (file.isDirectory() && file.getName().endsWith(V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION)) {
FileUtils.deleteDirectory(file);
}
}
@@ -222,7 +222,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
private void copyLuceneTextIndexIfExists(File segmentDirectory, File v3Dir)
throws IOException {
// TODO: see if this can be done by reusing some existing methods
- String suffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ String suffix = V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION;
File[] textIndexFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -243,7 +243,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
// to moving the lucene text index files, we need to move the
// docID mapping/cache file created by us in v1/v2 during an earlier
// load of the segment.
- String docIDFileSuffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ String docIDFileSuffix = V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
File[] textIndexDocIdMappingFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
index 683227031f..d04ce7bc97 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
@@ -20,8 +20,8 @@
package org.apache.pinot.segment.local.segment.index.fst;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import java.io.IOException;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -64,7 +64,8 @@ public class FstIndexType extends AbstractIndexType<FstIndexConfig, TextIndexRea
implements ConfigurableFromIndexLoadingConfig<FstIndexConfig> {
public static final String INDEX_DISPLAY_NAME = "fst";
private static final List<String> EXTENSIONS =
- Collections.singletonList(V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
+ ImmutableList.of(V1Constants.Indexes.LUCENE_FST_INDEX_FILE_EXTENSION,
+ V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION);
protected FstIndexType() {
super(StandardIndexes.FST_ID);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
index e387ba3929..778e92db0a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
@@ -44,7 +44,7 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION;
/**
@@ -157,7 +157,7 @@ public class FSTIndexHandler extends BaseIndexHandler {
String segmentName = _segmentDirectory.getSegmentMetadata().getName();
String columnName = columnMetadata.getColumnName();
File inProgress = new File(indexDir, columnName + ".fst.inprogress");
- File fstIndexFile = new File(indexDir, columnName + FST_INDEX_FILE_EXTENSION);
+ File fstIndexFile = new File(indexDir, columnName + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
if (!inProgress.exists()) {
// Create a marker file.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
index f1b5789fd5..0d4646e1da 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
@@ -84,9 +84,9 @@ public class LuceneTextIndexReader implements TextIndexReader {
// TODO: consider using a threshold of num docs per segment to decide between building
// mapping file upfront on segment load v/s on-the-fly during query processing
_docIdTranslator = new DocIdTranslator(indexDir, _column, numDocs, _indexSearcher);
- _standardAnalyzer = TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(
- config.getStopWordsInclude(), config.getStopWordsExclude()
- );
+ _standardAnalyzer = TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(config.getStopWordsInclude(),
+ config.getStopWordsExclude());
+ LOGGER.info("Successfully read lucene index for {} from {}", _column, indexDir);
} catch (Exception e) {
LOGGER.error("Failed to instantiate Lucene text index reader for column {}, exception {}", column,
e.getMessage());
@@ -99,15 +99,15 @@ public class LuceneTextIndexReader implements TextIndexReader {
* we load the text index (per column if it exists) and store
* the reference in {@link PhysicalColumnIndexContainer}
* similar to how it is done for other types of indexes.
- * @param column column name
+ *
+ * @param column column name
* @param indexDir segment index directory
- * @param numDocs number of documents in the segment
+ * @param numDocs number of documents in the segment
*/
public LuceneTextIndexReader(String column, File indexDir, int numDocs,
@Nullable Map<String, String> textIndexProperties) {
- this(column, indexDir, numDocs, new TextIndexConfigBuilder(FSTType.LUCENE)
- .withProperties(textIndexProperties)
- .build());
+ this(column, indexDir, numDocs,
+ new TextIndexConfigBuilder(FSTType.LUCENE).withProperties(textIndexProperties).build());
}
/**
@@ -120,6 +120,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
* CASE 2: However, if IndexLoadingConfig doesn't specify the segment version to load or if the specified
* version is same as the on-disk version of the segment, then ImmutableSegmentLoader will load
* whatever the version of segment is on disk.
+ *
* @param segmentIndexDir top-level segment index directory
* @return text index file
*/
@@ -163,6 +164,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
* When we destroy the loaded ImmutableSegment, all the indexes
* (for each column) are destroyed and as part of that
* we release the text index
+ *
* @throws IOException
*/
@Override
@@ -188,7 +190,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
throws Exception {
int length = Integer.BYTES * numDocs;
File docIdMappingFile = new File(SegmentDirectoryPaths.findSegmentDirectory(segmentIndexDir),
- column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
// The mapping is local to a segment. It is created on the server during segment load.
// Unless we are running Pinot on Solaris/SPARC, the underlying architecture is
// LITTLE_ENDIAN (Linux/x86). So use that as byte order.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
index bfd1732439..4ad6eb3a94 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
@@ -74,7 +74,8 @@ public class TextIndexType extends AbstractIndexType<TextIndexConfig, TextIndexR
// TODO: Should V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION be added here?
private static final List<String> EXTENSIONS = Lists.newArrayList(
V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION,
- V1Constants.Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION
+ V1Constants.Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION,
+ V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION
);
protected TextIndexType() {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
index aa474743c6..2d0510c486 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
@@ -47,6 +47,10 @@ public class TextIndexUtils {
FileUtils.deleteQuietly(luceneIndexFile);
File luceneMappingFile = new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
FileUtils.deleteQuietly(luceneMappingFile);
+ File luceneV9IndexFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
+ FileUtils.deleteQuietly(luceneV9IndexFile);
+ File luceneV9MappingFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ FileUtils.deleteQuietly(luceneV9MappingFile);
// Remove the native index file
File nativeIndexFile = new File(segDir, column + Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION);
@@ -54,8 +58,9 @@ public class TextIndexUtils {
}
static boolean hasTextIndex(File segDir, String column) {
- return new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists() || new File(segDir,
- column + Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION).exists();
+ return new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists()
+ || new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists()
+ || new File(segDir, column + Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION).exists();
}
public static boolean isFstTypeNative(@Nullable Map<String, String> textIndexProperties) {
@@ -108,7 +113,7 @@ public class TextIndexUtils {
}
public static StandardAnalyzer getStandardAnalyzerWithCustomizedStopWords(@Nullable List<String> stopWordsInclude,
- @Nullable List<String> stopWordsExclude) {
+ @Nullable List<String> stopWordsExclude) {
HashSet<String> stopWordSet = LuceneTextIndexCreator.getDefaultEnglishStopWordsSet();
if (stopWordsInclude != null) {
stopWordSet.addAll(stopWordsInclude);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
index 1fb58b56bc..1f69bc50f1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
@@ -47,7 +47,7 @@ public class NativeFSTIndexCreator implements FSTIndexCreator {
* @param sortedEntries Sorted entries of the unique values of the column.
*/
public NativeFSTIndexCreator(File indexDir, String columnName, String[] sortedEntries) {
- _fstIndexFile = new File(indexDir, columnName + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
+ _fstIndexFile = new File(indexDir, columnName + V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION);
_fstBuilder = new FSTBuilder();
_dictId = 0;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
index c12a969037..b9c568000e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/LuceneFSTIndexCreatorTest.java
@@ -32,7 +32,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION;
public class LuceneFSTIndexCreatorTest {
@@ -62,7 +62,7 @@ public class LuceneFSTIndexCreatorTest {
LuceneFSTIndexCreator creator = new LuceneFSTIndexCreator(
INDEX_DIR, "testFSTColumn", uniqueValues);
creator.seal();
- File fstFile = new File(INDEX_DIR, "testFSTColumn" + FST_INDEX_FILE_EXTENSION);
+ File fstFile = new File(INDEX_DIR, "testFSTColumn" + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
PinotDataBuffer pinotDataBuffer =
PinotDataBuffer.mapFile(fstFile, true, 0, fstFile.length(), ByteOrder.BIG_ENDIAN, "fstIndexFile");
LuceneFSTIndexReader reader = new LuceneFSTIndexReader(pinotDataBuffer);
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/NativeFSTIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/NativeFSTIndexCreatorTest.java
index cc4f153d8e..d77268ef28 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/NativeFSTIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/NativeFSTIndexCreatorTest.java
@@ -29,7 +29,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION;
public class NativeFSTIndexCreatorTest {
@@ -59,7 +59,7 @@ public class NativeFSTIndexCreatorTest {
creator.seal();
}
- File fstFile = new File(INDEX_DIR, "testFSTColumn" + FST_INDEX_FILE_EXTENSION);
+ File fstFile = new File(INDEX_DIR, "testFSTColumn" + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
try (PinotDataBuffer dataBuffer = PinotDataBuffer.mapReadOnlyBigEndianFile(fstFile);
NativeFSTIndexReader reader = new NativeFSTIndexReader(dataBuffer)) {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index c9db1e3148..bc64d539b0 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -63,7 +63,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import org.testng.collections.Lists;
-import static org.apache.pinot.segment.spi.V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
+import static org.apache.pinot.segment.spi.V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -330,7 +330,7 @@ public class LoaderTest {
fstIndexFile = SegmentDirectoryPaths.findFSTIndexIndexFile(_indexDir, FST_INDEX_COL_NAME);
Assert.assertNotNull(fstIndexFile);
Assert.assertFalse(fstIndexFile.isDirectory());
- Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + FST_INDEX_FILE_EXTENSION);
+ Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
Assert.assertEquals(fstIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -349,7 +349,7 @@ public class LoaderTest {
fstIndexFile = SegmentDirectoryPaths.findFSTIndexIndexFile(_indexDir, FST_INDEX_COL_NAME);
Assert.assertNotNull(fstIndexFile);
Assert.assertFalse(fstIndexFile.isDirectory());
- Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + FST_INDEX_FILE_EXTENSION);
+ Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
Assert.assertEquals(fstIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -366,7 +366,7 @@ public class LoaderTest {
fstIndexFile = SegmentDirectoryPaths.findFSTIndexIndexFile(_indexDir, FST_INDEX_COL_NAME);
Assert.assertNotNull(fstIndexFile);
Assert.assertFalse(fstIndexFile.isDirectory());
- Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + FST_INDEX_FILE_EXTENSION);
+ Assert.assertEquals(fstIndexFile.getName(), FST_INDEX_COL_NAME + LUCENE_V9_FST_INDEX_FILE_EXTENSION);
Assert.assertEquals(fstIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -579,7 +579,7 @@ public class LoaderTest {
Assert.assertNotNull(textIndexFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
// CASE 1: don't set the segment version to load in IndexLoadingConfig
@@ -605,10 +605,10 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -633,10 +633,10 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -660,7 +660,7 @@ public class LoaderTest {
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertFalse(textIndexDocIdMappingFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
// CASE 1: don't set the segment version to load in IndexLoadingConfig
@@ -683,10 +683,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -709,10 +709,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -735,10 +735,10 @@ public class LoaderTest {
Assert.assertNotNull(textIndexDocIdMappingFile);
Assert.assertTrue(textIndexFile.isDirectory());
Assert.assertEquals(textIndexFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index cfe3a9aadd..8e549d2ba3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -1415,8 +1415,8 @@ public class SegmentPreProcessorTest {
// V1 use separate file for each column index.
File iiFile = new File(_indexDir, strColumn + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
File rgFile = new File(_indexDir, strColumn + V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION);
- File txtFile = new File(_indexDir, strColumn + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION);
- File fstFile = new File(_indexDir, strColumn + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION);
+ File txtFile = new File(_indexDir, strColumn + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
+ File fstFile = new File(_indexDir, strColumn + V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION);
File bfFile = new File(_indexDir, strColumn + V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION);
assertFalse(iiFile.exists());
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index 7cc4b7c70c..e115414b3e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -230,12 +230,13 @@ public class FilePerIndexDirectoryTest {
// Both files for TextIndex should be removed.
fpi.removeIndex("foo", StandardIndexes.text());
- assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists());
+ assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
- assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists());
- assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
+ assertTrue(
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 32feaa0177..79778c8104 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -263,12 +263,13 @@ public class SingleFileIndexDirectoryTest {
// Both files for TextIndex should be removed.
sfd.removeIndex("foo", StandardIndexes.text());
- assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists());
+ assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
- assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists());
- assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
+ assertTrue(
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 8900510ab6..218d174614 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -44,14 +44,17 @@ public class V1Constants {
public static final String UNSORTED_MV_FORWARD_INDEX_FILE_EXTENSION = ".mv.fwd";
public static final String BITMAP_INVERTED_INDEX_FILE_EXTENSION = ".bitmap.inv";
public static final String BITMAP_RANGE_INDEX_FILE_EXTENSION = ".bitmap.range";
- public static final String FST_INDEX_FILE_EXTENSION = ".lucene.v9.fst";
public static final String JSON_INDEX_FILE_EXTENSION = ".json.idx";
public static final String NATIVE_TEXT_INDEX_FILE_EXTENSION = ".nativetext.idx";
public static final String H3_INDEX_FILE_EXTENSION = ".h3.idx";
public static final String BLOOM_FILTER_FILE_EXTENSION = ".bloom";
public static final String NULLVALUE_VECTOR_FILE_EXTENSION = ".bitmap.nullvalue";
- public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.v9.mapping";
- public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.v9.index";
+ public static final String LUCENE_FST_INDEX_FILE_EXTENSION = ".lucene.fst";
+ public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
+ public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
+ public static final String LUCENE_V9_FST_INDEX_FILE_EXTENSION = ".lucene.v9.fst";
+ public static final String LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.v9.mapping";
+ public static final String LUCENE_V9_TEXT_INDEX_FILE_EXTENSION = ".lucene.v9.index";
}
public static class MetadataKeys {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
index 61686c38f7..98b60df91e 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
@@ -79,8 +79,13 @@ public class SegmentDirectoryPaths {
*/
@Nullable
public static File findTextIndexIndexFile(File indexDir, String column) {
- String luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
- return findFormatFile(indexDir, luceneIndexDirectory);
+ String luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION;
+ File indexFormatFile = findFormatFile(indexDir, luceneIndexDirectory);
+ if (indexFormatFile == null) {
+ luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION;
+ indexFormatFile = findFormatFile(indexDir, luceneIndexDirectory);
+ }
+ return indexFormatFile;
}
/**
@@ -96,15 +101,25 @@ public class SegmentDirectoryPaths {
}
public static File findFSTIndexIndexFile(File indexDir, String column) {
- String luceneIndexDirectory = column + V1Constants.Indexes.FST_INDEX_FILE_EXTENSION;
- return findFormatFile(indexDir, luceneIndexDirectory);
+ String luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_V9_FST_INDEX_FILE_EXTENSION;
+ File formatFile = findFormatFile(indexDir, luceneIndexDirectory);
+ if (formatFile == null) {
+ luceneIndexDirectory = column + V1Constants.Indexes.LUCENE_FST_INDEX_FILE_EXTENSION;
+ formatFile = findFormatFile(indexDir, luceneIndexDirectory);
+ }
+ return formatFile;
}
@Nullable
@VisibleForTesting
public static File findTextIndexDocIdMappingFile(File indexDir, String column) {
- String file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
- return findFormatFile(indexDir, file);
+ String file = column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ File formatFile = findFormatFile(indexDir, file);
+ if (formatFile == null) {
+ file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ formatFile = findFormatFile(indexDir, file);
+ }
+ return formatFile;
}
/**
diff --git a/pinot-tools/src/main/resources/examples/batch/fineFoodReviews/fineFoodReviews_offline_table_config.json b/pinot-tools/src/main/resources/examples/batch/fineFoodReviews/fineFoodReviews_offline_table_config.json
index 886c70ea80..acc50ae301 100644
--- a/pinot-tools/src/main/resources/examples/batch/fineFoodReviews/fineFoodReviews_offline_table_config.json
+++ b/pinot-tools/src/main/resources/examples/batch/fineFoodReviews/fineFoodReviews_offline_table_config.json
@@ -1,4 +1,15 @@
{
+ "fieldConfigList": [
+ {
+ "encodingType": "RAW",
+ "indexType": "TEXT",
+ "name": "Text",
+ "properties": {
+ "deriveNumDocsPerChunkForRawIndex": "true",
+ "rawIndexWriterVersion": "3"
+ }
+ }
+ ],
"tableName": "fineFoodReviews",
"tableType": "OFFLINE",
"segmentsConfig": {
@@ -11,6 +22,7 @@
},
"tableIndexConfig": {
"loadMode": "MMAP",
+ "noDictionaryColumns": ["Text"],
"invertedIndexColumns": [
]
},
diff --git a/pom.xml b/pom.xml
index 29a739be48..fac14ba080 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1403,6 +1403,30 @@
<artifactId>woodstox-core</artifactId>
<version>${woodstox.version}</version>
</dependency>
+
+ <!-- Lucene dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analysis-common</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <!-- Lucene dependencies end -->
+
</dependencies>
</dependencyManagement>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org