You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/11/07 08:33:13 UTC
(pinot) 03/03: adding lucene 80 compatible test
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 06b58d4dcd723923f7fb61593115b6b44114edd0
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Thu Nov 2 20:03:21 2023 -0700
adding lucene 80 compatible test
---
.../converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../index/readers/text/LuceneTextIndexReader.java | 2 +-
.../local/segment/store/TextIndexUtils.java | 2 +-
.../local/segment/index/loader/LoaderTest.java | 10 +++---
.../text/LuceneTextIndexCompatibleTest.java | 36 +++++++++++++++++++++
.../segment/store/FilePerIndexDirectoryTest.java | 4 +--
.../store/SingleFileIndexDirectoryTest.java | 4 +--
.../data/lucene_80_index/Text.lucene.index/_0.cfe | Bin 0 -> 299 bytes
.../data/lucene_80_index/Text.lucene.index/_0.cfs | Bin 0 -> 136123 bytes
.../data/lucene_80_index/Text.lucene.index/_0.si | Bin 0 -> 370 bytes
.../lucene_80_index/Text.lucene.index/segments_1 | Bin 0 -> 137 bytes
.../lucene_80_index/Text.lucene.index/write.lock | 0
.../org/apache/pinot/segment/spi/V1Constants.java | 1 -
.../segment/spi/store/SegmentDirectoryPaths.java | 9 ++----
pom.xml | 1 +
15 files changed, 51 insertions(+), 20 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index d6cc8b1515..a65b58fd32 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -243,7 +243,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
// to moving the lucene text index files, we need to move the
// docID mapping/cache file created by us in v1/v2 during an earlier
// load of the segment.
- String docIDFileSuffix = V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ String docIDFileSuffix = V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
File[] textIndexDocIdMappingFiles = segmentDirectory.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
index 0d4646e1da..e249412e6b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
@@ -190,7 +190,7 @@ public class LuceneTextIndexReader implements TextIndexReader {
throws Exception {
int length = Integer.BYTES * numDocs;
File docIdMappingFile = new File(SegmentDirectoryPaths.findSegmentDirectory(segmentIndexDir),
- column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
// The mapping is local to a segment. It is created on the server during segment load.
// Unless we are running Pinot on Solaris/SPARC, the underlying architecture is
// LITTLE_ENDIAN (Linux/x86). So use that as byte order.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
index 2d0510c486..235025acde 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
@@ -49,7 +49,7 @@ public class TextIndexUtils {
FileUtils.deleteQuietly(luceneMappingFile);
File luceneV9IndexFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
FileUtils.deleteQuietly(luceneV9IndexFile);
- File luceneV9MappingFile = new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ File luceneV9MappingFile = new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
FileUtils.deleteQuietly(luceneV9MappingFile);
// Remove the native index file
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index bc64d539b0..6c73f20e17 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -608,7 +608,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -636,7 +636,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
@@ -686,7 +686,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -712,7 +712,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), new SegmentMetadataImpl(_indexDir).getName());
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
new SegmentMetadataImpl(_indexDir).getName());
indexSegment.destroy();
@@ -738,7 +738,7 @@ public class LoaderTest {
TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION);
Assert.assertEquals(textIndexFile.getParentFile().getName(), SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
Assert.assertEquals(textIndexDocIdMappingFile.getName(),
- TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
+ TEXT_INDEX_COL_NAME + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION);
Assert.assertEquals(textIndexDocIdMappingFile.getParentFile().getName(),
SegmentDirectoryPaths.V3_SUBDIRECTORY_NAME);
indexSegment.destroy();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java
new file mode 100644
index 0000000000..5ac2c6a17a
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexCompatibleTest.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers.text;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class LuceneTextIndexCompatibleTest {
+
+ @Test
+ public void testLucene80IndexReader() {
+ File indexPath =
+ new File(LuceneTextIndexCompatibleTest.class.getClassLoader().getResource("data/lucene_80_index").getPath());
+ LuceneTextIndexReader lucene80Index = new LuceneTextIndexReader("Text", indexPath, 1000, ImmutableMap.of());
+ Assert.assertNotNull(lucene80Index);
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index e115414b3e..ca933e01b3 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -232,11 +232,11 @@ public class FilePerIndexDirectoryTest {
fpi.removeIndex("foo", StandardIndexes.text());
assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertTrue(
- new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 79778c8104..3c8ba99688 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -265,11 +265,11 @@ public class SingleFileIndexDirectoryTest {
sfd.removeIndex("foo", StandardIndexes.text());
assertFalse(new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertFalse(
- new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "foo" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
}
assertTrue(new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists());
assertTrue(
- new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
+ new File(TEMP_DIR, "bar" + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION).exists());
// Read indices back and check the content.
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR, _segmentMetadata, ReadMode.mmap)) {
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe
new file mode 100644
index 0000000000..6d4909e1bd
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfe differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs
new file mode 100644
index 0000000000..f57db05798
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.cfs differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si
new file mode 100644
index 0000000000..e92713aad5
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/_0.si differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1 b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1
new file mode 100644
index 0000000000..d67a947f7a
Binary files /dev/null and b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/segments_1 differ
diff --git a/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock b/pinot-segment-local/src/test/resources/data/lucene_80_index/Text.lucene.index/write.lock
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 218d174614..887bd87ff5 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -53,7 +53,6 @@ public class V1Constants {
public static final String LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.mapping";
public static final String LUCENE_TEXT_INDEX_FILE_EXTENSION = ".lucene.index";
public static final String LUCENE_V9_FST_INDEX_FILE_EXTENSION = ".lucene.v9.fst";
- public static final String LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION = ".lucene.v9.mapping";
public static final String LUCENE_V9_TEXT_INDEX_FILE_EXTENSION = ".lucene.v9.index";
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
index 98b60df91e..2f1cf67b81 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/store/SegmentDirectoryPaths.java
@@ -113,13 +113,8 @@ public class SegmentDirectoryPaths {
@Nullable
@VisibleForTesting
public static File findTextIndexDocIdMappingFile(File indexDir, String column) {
- String file = column + V1Constants.Indexes.LUCENE_V9_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
- File formatFile = findFormatFile(indexDir, file);
- if (formatFile == null) {
- file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
- formatFile = findFormatFile(indexDir, file);
- }
- return formatFile;
+ String file = column + V1Constants.Indexes.LUCENE_TEXT_INDEX_DOCID_MAPPING_FILE_EXTENSION;
+ return findFormatFile(indexDir, file);
}
/**
diff --git a/pom.xml b/pom.xml
index fac14ba080..7f236c1ae7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2028,6 +2028,7 @@
<exclude>**/*.ser</exclude>
<exclude>**/*.v1</exclude>
<exclude>**/*.v2</exclude>
+ <exclude>**/Text.lucene.index/**</exclude>
<exclude>**/appAssemblerScriptTemplate</exclude>
<exclude>doap_Pinot.rdf</exclude>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org