You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/12/05 23:23:29 UTC
(pinot) branch master updated: Fix compilation issue for lucene analyzer class change (#12101)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9013e724a7 Fix compilation issue for lucene analyzer class change (#12101)
9013e724a7 is described below
commit 9013e724a724329cff5480382945a9936e6a556c
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Tue Dec 5 15:23:24 2023 -0800
Fix compilation issue for lucene analyzer class change (#12101)
---
.../creator/impl/text/LuceneTextIndexCreator.java | 9 +++------
.../index/readers/text/LuceneTextIndexReader.java | 5 ++---
.../local/segment/store/TextIndexUtils.java | 23 +++++++++-------------
.../pinot/segment/spi/index/TextIndexConfig.java | 11 ++++-------
4 files changed, 18 insertions(+), 30 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 59d1297f7e..f14cf62bc6 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -41,8 +41,6 @@ import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.TextIndexConfig;
import org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCreator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
@@ -51,7 +49,6 @@ import org.slf4j.LoggerFactory;
* and realtime from {@link RealtimeLuceneTextIndex}
*/
public class LuceneTextIndexCreator extends AbstractTextIndexCreator {
- private static final Logger LOGGER = LoggerFactory.getLogger(LuceneTextIndexCreator.class);
public static final String LUCENE_INDEX_DOC_ID_COLUMN_NAME = "DocID";
private final String _textColumn;
@@ -104,10 +101,10 @@ public class LuceneTextIndexCreator extends AbstractTextIndexCreator {
Analyzer luceneAnalyzer;
if (luceneAnalyzerClass.isEmpty() || luceneAnalyzerClass.equals(StandardAnalyzer.class.getName())) {
- luceneAnalyzer = TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(
- config.getStopWordsInclude(), config.getStopWordsExclude());
+ luceneAnalyzer = TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(config.getStopWordsInclude(),
+ config.getStopWordsExclude());
} else {
- luceneAnalyzer = TextIndexUtils.getAnalyzerFromFQCN(luceneAnalyzerClass);
+ luceneAnalyzer = TextIndexUtils.getAnalyzerFromClassName(luceneAnalyzerClass);
}
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(luceneAnalyzer);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
index abe56e6736..9b971d5142 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexReader.java
@@ -87,9 +87,8 @@ public class LuceneTextIndexReader implements TextIndexReader {
_docIdTranslator = new DocIdTranslator(indexDir, _column, numDocs, _indexSearcher);
String luceneAnalyzerClass = config.getLuceneAnalyzerClass();
_analyzer = luceneAnalyzerClass.equals(StandardAnalyzer.class.getName())
- ? TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(
- config.getStopWordsInclude(), config.getStopWordsExclude())
- : TextIndexUtils.getAnalyzerFromFQCN(luceneAnalyzerClass);
+ ? TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(config.getStopWordsInclude(),
+ config.getStopWordsExclude()) : TextIndexUtils.getAnalyzerFromClassName(luceneAnalyzerClass);
LOGGER.info("Successfully read lucene index for {} from {}", _column, indexDir);
} catch (Exception e) {
LOGGER.error("Failed to instantiate Lucene text index reader for column {}, exception {}", column,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
index 883ad47c4d..caa47adff7 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/store/TextIndexUtils.java
@@ -25,7 +25,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
@@ -59,9 +58,11 @@ public class TextIndexUtils {
}
static boolean hasTextIndex(File segDir, String column) {
+ //@formatter:off
return new File(segDir, column + Indexes.LUCENE_TEXT_INDEX_FILE_EXTENSION).exists()
|| new File(segDir, column + Indexes.LUCENE_V9_TEXT_INDEX_FILE_EXTENSION).exists()
|| new File(segDir, column + Indexes.NATIVE_TEXT_INDEX_FILE_EXTENSION).exists();
+ //@formatter:on
}
public static boolean isFstTypeNative(@Nullable Map<String, String> textIndexProperties) {
@@ -80,43 +81,37 @@ public class TextIndexUtils {
return SegmentDirectoryPaths.findTextIndexIndexFile(indexDir, column) != null ? FSTType.LUCENE : FSTType.NATIVE;
}
- @Nonnull
public static List<String> extractStopWordsInclude(String colName,
Map<String, Map<String, String>> columnProperties) {
return extractStopWordsInclude(columnProperties.getOrDefault(colName, null));
}
- @Nonnull
public static List<String> extractStopWordsExclude(String colName,
Map<String, Map<String, String>> columnProperties) {
return extractStopWordsExclude(columnProperties.getOrDefault(colName, null));
}
- @Nonnull
public static List<String> extractStopWordsInclude(Map<String, String> columnProperty) {
return parseEntryAsString(columnProperty, FieldConfig.TEXT_INDEX_STOP_WORD_INCLUDE_KEY);
}
- @Nonnull
public static List<String> extractStopWordsExclude(Map<String, String> columnProperty) {
return parseEntryAsString(columnProperty, FieldConfig.TEXT_INDEX_STOP_WORD_EXCLUDE_KEY);
}
- @Nonnull
- private static List<String> parseEntryAsString(@Nullable Map<String, String> columnProperties,
- String stopWordKey) {
+ private static List<String> parseEntryAsString(@Nullable Map<String, String> columnProperties, String stopWordKey) {
if (columnProperties == null) {
- return Collections.EMPTY_LIST;
+ return Collections.emptyList();
}
String includeWords = columnProperties.getOrDefault(stopWordKey, "");
- return Arrays.stream(includeWords.split(FieldConfig.TEXT_INDEX_STOP_WORD_SEPERATOR))
- .map(String::trim).collect(Collectors.toList());
+ return Arrays.stream(includeWords.split(FieldConfig.TEXT_INDEX_STOP_WORD_SEPERATOR)).map(String::trim)
+ .collect(Collectors.toList());
}
- public static Analyzer getAnalyzerFromClassName(String luceneAnalyzerClass) throws
- ReflectiveOperationException {
+ public static Analyzer getAnalyzerFromClassName(String luceneAnalyzerClass)
+ throws ReflectiveOperationException {
// Support instantiation with default constructor for now unless customized
- return (Analyzer) Class.forName(luceneAnalyzerFQCN).getConstructor().newInstance();
+ return (Analyzer) Class.forName(luceneAnalyzerClass).getConstructor().newInstance();
}
public static StandardAnalyzer getStandardAnalyzerWithCustomizedStopWords(@Nullable List<String> stopWordsInclude,
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
index ff8e851084..0b31e70e1e 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
@@ -50,9 +50,7 @@ public class TextIndexConfig extends IndexConfig {
private final String _luceneAnalyzerClass;
@JsonCreator
- public TextIndexConfig(
- @JsonProperty("disabled") Boolean disabled,
- @JsonProperty("fst") FSTType fstType,
+ public TextIndexConfig(@JsonProperty("disabled") Boolean disabled, @JsonProperty("fst") FSTType fstType,
@JsonProperty("rawValue") @Nullable Object rawValueForTextIndex,
@JsonProperty("queryCache") boolean enableQueryCache,
@JsonProperty("useANDForMultiTermQueries") boolean useANDForMultiTermQueries,
@@ -60,8 +58,7 @@ public class TextIndexConfig extends IndexConfig {
@JsonProperty("stopWordsExclude") List<String> stopWordsExclude,
@JsonProperty("luceneUseCompoundFile") Boolean luceneUseCompoundFile,
@JsonProperty("luceneMaxBufferSizeMB") Integer luceneMaxBufferSizeMB,
- @JsonProperty("luceneAnalyzerClass") String luceneAnalyzerClass
- ) {
+ @JsonProperty("luceneAnalyzerClass") String luceneAnalyzerClass) {
super(disabled);
_fstType = fstType;
_rawValueForTextIndex = rawValueForTextIndex;
@@ -74,7 +71,7 @@ public class TextIndexConfig extends IndexConfig {
_luceneMaxBufferSizeMB =
luceneMaxBufferSizeMB == null ? LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB : luceneMaxBufferSizeMB;
_luceneAnalyzerClass = (luceneAnalyzerClass == null || luceneAnalyzerClass.isEmpty())
- ? FieldConfig.TEXT_INDEX_DEFAULT_LUCENE_ANALYZER_CLASS : luceneAnalyzerClass;
+ ? FieldConfig.TEXT_INDEX_DEFAULT_LUCENE_ANALYZER_CLASS : luceneAnalyzerClass;
}
public FSTType getFstType() {
@@ -189,7 +186,7 @@ public class TextIndexConfig extends IndexConfig {
}
public AbstractBuilder withLuceneAnalyzerClass(String luceneAnalyzerClass) {
- _luceneAnalyzerClass = luceneAnalyzerFQCN;
+ _luceneAnalyzerClass = luceneAnalyzerClass;
return this;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org