You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by GitBox <gi...@apache.org> on 2022/05/04 10:30:12 UTC
[GitHub] [pinot] atris opened a new pull request, #8636: Introduce Real Time Native Text Index
atris opened a new pull request, #8636:
URL: https://github.com/apache/pinot/pull/8636
This commit introduces the real time native text index. The text index is real time i.e. as data comes in, it is available for search.
There is no refresh required. The index works with TEXT_CONTAINS.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r867192427
##########
pinot-core/src/test/java/org/apache/pinot/queries/RealTimeNativeVsLuceneTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class RealTimeNativeVsLuceneTest extends BaseQueriesTest {
Review Comment:
Rename
```suggestion
public class NativeVsLuceneMutableTextIndexTest extends BaseQueriesTest {
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+public class NativeMutableTextIndexConcurrentTest {
+ private ExecutorService _threadPool;
+ private Set<String> _resultSet;
+
+ @BeforeClass
+ private void setup() {
+ _threadPool = Executors.newFixedThreadPool(10);
+ _resultSet = new ConcurrentSkipListSet<>();
+ }
+
+ @AfterClass
+ private void shutDown() {
+ _threadPool.shutdown();
+ }
+
+ @Test
+ public void testConcurrentWriteAndRead()
+ throws InterruptedException, IOException {
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ List<String> words = new ArrayList<>();
+ words.add("ab");
+ words.add("abba");
+ words.add("aba");
+ words.add("bab");
+ words.add("cdd");
+ words.add("efg");
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, words, 20, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
Review Comment:
We want to throw the exception (wrap it in `RuntimeException`) instead of just printing the stack trace. Same for other places
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/mutablefst/MutableFSTConcurrentTest.java:
##########
@@ -102,8 +100,54 @@ public void testConcurrentWriteAndRead()
assertTrue("efg not found in result set", _resultSet.contains("efg"));
}
- private void performReads(MutableFST fst, List<String> words, int count,
- long sleepTime)
+ @Test
+ public void testConcurrentLongWriteAndRead()
+ throws InterruptedException {
+ MutableFST mutableFST = new MutableFSTImpl();
+ List<String> words = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+
+ mutableFST.addPath("ab", 1);
+
+ List<Pair<String, Integer>> wordsWithMetadata = new ArrayList<>();
+
+ // Add some write pressure
+ wordsWithMetadata.add(Pair.of("egegdgrbsbrsegzgzegzegegjntnmtj", 2));
+ wordsWithMetadata.add(Pair.of("hrwbwefweg4wreghrtbrassregfesfefefefzew4ere", 2));
+ wordsWithMetadata.add(Pair.of("easzegfegrertegbxzzez3erfezgzeddzdewstfefed", 2));
+ wordsWithMetadata.add(Pair.of("tjntrhndsrsgezgrsxzetgteszetgezfzezedrefzdzdzdzdz", 2));
+ wordsWithMetadata.add(Pair.of("abacxcvbnmlkjjhgfsaqwertyuioopzxcvbnmllkjshfgsfawieeiuefgeurfeoafa", 2));
+
+ words.add("abacxcvbnmlkjjhgfsaqwertyuioopzxcvbnmllkjshfgsfawieeiuefgeurfeoafa");
+
+ _threadPool.submit(() -> {
+ try {
+ performReads(mutableFST, words, 10, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
Review Comment:
Wrap it in `RuntimeException`
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndexReaderWriterTest.java:
##########
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class NativeMutableTextIndexReaderWriterTest {
+
+ @Test
+ public void testIndexWriterReader()
+ throws IOException {
+ String[] uniqueValues = new String[4];
+ uniqueValues[0] = "hello-world";
+ uniqueValues[1] = "hello-world123";
+ uniqueValues[2] = "still";
+ uniqueValues[3] = "zoobar";
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ for (int i = 0; i < 4; i++) {
+ textIndex.add(uniqueValues[i]);
+ }
+
+ int[] matchedDocIds = textIndex.getDocIds("hello.*").toArray();
+ Assert.assertEquals(2, matchedDocIds.length);
Review Comment:
(minor) Import `Assert.assertEquals`
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndex.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeMutableTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeMutableTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
Did some research, and I think `StandardAnalyzer` can be defined as constant. It should be thread safe in single writer context
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NativeRealtimeClusterIntegrationTest.java:
##########
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.util.TestUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+
+public class NativeRealtimeClusterIntegrationTest extends BaseClusterIntegrationTest {
Review Comment:
Let's merge this into the existing realtime text index test
##########
pinot-core/src/test/java/org/apache/pinot/queries/RealTimeNativeVsLuceneTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class RealTimeNativeVsLuceneTest extends BaseQueriesTest {
Review Comment:
This is not really a queries test. We should either make it a unit test for the text index, or make it similar to `NativeAndLuceneComparisonTest`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] kkrugler commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
kkrugler commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866241674
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
I think you can reuse an `Analyzer`, though it's a bit tricky. Here's code we use to extract terms from text, by wrapping a Lucene Analyzer (note pending optimization to have a reusable `Reader`):
``` java
public class TermsParser implements Closeable {
private Analyzer _analyzer;
// These need to be thread local, since Lucene has them as thread local as well.
private ThreadLocal<TokenStream> _tokenStream = new ThreadLocal<TokenStream>();
private ThreadLocal<CharTermAttribute> _token = new ThreadLocal<CharTermAttribute>();
/**
* Set up an Analyzer to use while parsing text.
*
*
* @param termsAnalyzerFactory
*/
public TermsParser(Analyzer analyzer) {
_analyzer = analyzer;
}
public Iterable<String> parse(CharSequence in) {
// TODO make reader reusable, so we can reset it
return parse(new CharSequenceReader(in));
}
public Iterable<String> parse(Reader in) {
try {
if (_tokenStream.get() != null) {
_tokenStream.get().close();
}
_tokenStream.set(_analyzer.tokenStream("content", in));
_tokenStream.get().reset();
if (_token.get() == null) {
_token.set((CharTermAttribute) _tokenStream.get().addAttribute(CharTermAttribute.class));
}
} catch (IOException e) {
throw new RuntimeException("Impossible exception", e);
}
_token.get().setEmpty();
return new Iterable<String>() {
@Override
public Iterator<String> iterator() {
return new TokenIterator(_tokenStream.get(), _token.get());
}
};
}
private static class TokenIterator implements Iterator<String> {
private TokenStream _tokenStream;
private CharTermAttribute _token;
public TokenIterator(TokenStream tokenStream, CharTermAttribute token) {
_tokenStream = tokenStream;
_token = token;
}
@Override
public boolean hasNext() {
if (_token.length() == 0) {
try {
_tokenStream.incrementToken();
} catch (IOException e) {
throw new RuntimeException("Impossible exception", e);
}
}
return _token.length() > 0;
}
@Override
public String next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
String result = new String(_token.buffer(), 0, _token.length());
_token.setEmpty();
return result;
}
}
@Override
public void close() {
_analyzer.close();
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866318887
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeRealTimeTextIndex.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeRealTimeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeRealTimeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ _writeLock.lock();
+ try {
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.computeIfAbsent(token, k -> {
Review Comment:
```suggestion
int currentDictId = _termToDictIdMapping.computeIntIfAbsent(token, k -> {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on PR #8636:
URL: https://github.com/apache/pinot/pull/8636#issuecomment-1121792892
@Jackie-Jiang Updated the PR.
Regards to the Analyser, I looked at the core Lucene class and also did some benchmarking -- due to the fact that only one thread is writing to a single mutable FST at any given point of time, I was not able to see any significant overhead due to the analyser refresh. However, I have attuned the code now per your comments.
Also, regards to the exceptions, I have been actively trying to limit the exceptions raised to the ones which are significant to the index -- for eg, I do not feel its worth failing the test if a thread is interrupted, or the Lucene SearcherManager failed to refresh. However, again, I have attuned the code now per your comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] yupeng9 commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
yupeng9 commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r865140193
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+
+public class RealtimeNativeTextIndexConcurrentTest {
+ private ExecutorService _threadPool;
+ private Set<String> _resultSet;
+
+ @BeforeClass
+ private void setup() {
+ _threadPool = Executors.newFixedThreadPool(10);
+ _resultSet = new ConcurrentSkipListSet<>();
+ }
+
+ @AfterClass
+ private void shutDown() {
+ _threadPool.shutdown();
+ }
+
+ @Test
+ public void testConcurrentWriteAndRead()
+ throws InterruptedException, IOException {
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ List<String> words = new ArrayList<>();
+ words.add("ab");
+ words.add("abba");
+ words.add("aba");
+ words.add("bab");
+ words.add("cdd");
+ words.add("efg");
+
+ try (RealtimeNativeTextIndex textIndex = new RealtimeNativeTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, words, 10, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, words, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), words.size());
+
+ assertTrue("ab not found in result set", _resultSet.contains("ab"));
+ assertTrue("abba not found in result set", _resultSet.contains("abba"));
+ assertTrue("aba not found in result set", _resultSet.contains("aba"));
+ assertTrue("bab not found in result set", _resultSet.contains("bab"));
+ assertTrue("cdd not found in result set", _resultSet.contains("cdd"));
+ assertTrue("efg not found in result set", _resultSet.contains("efg"));
+ }
+
+ @Test
+ public void testConcurrentWriteWithMultipleThreads()
+ throws InterruptedException, IOException {
+ List<String> firstThreadWords = new ArrayList<>();
+ List<String> secondThreadWords = new ArrayList<>();
+ List<String> mergedThreadWords = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ firstThreadWords.add("ab");
+ firstThreadWords.add("abba");
+ firstThreadWords.add("aba");
+ secondThreadWords.add("bab");
+ secondThreadWords.add("cdd");
+ secondThreadWords.add("efg");
+
+ mergedThreadWords.addAll(firstThreadWords);
+ mergedThreadWords.addAll(secondThreadWords);
+
+ try (RealtimeNativeTextIndex textIndex = new RealtimeNativeTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, mergedThreadWords, 10, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, firstThreadWords, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, secondThreadWords, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), mergedThreadWords.size());
+
+ assertTrue("ab not found in result set", _resultSet.contains("ab"));
Review Comment:
why not found?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] kkrugler commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
kkrugler commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866344056
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
@atris - overhead depends on the analyzer. E.g. I believe the `Kuromoji` (Japanese morphological analyzer) took significant time to initialize. Also this is the code we use when processing billions of records in a Flink workflow, so the overhead adds up (for us).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r868741367
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+public class NativeMutableTextIndexConcurrentTest {
+ private ExecutorService _threadPool;
+ private Set<String> _resultSet;
+
+ @BeforeClass
+ private void setup() {
+ _threadPool = Executors.newFixedThreadPool(10);
+ _resultSet = new ConcurrentSkipListSet<>();
+ }
+
+ @AfterClass
+ private void shutDown() {
+ _threadPool.shutdown();
+ }
+
+ @Test
+ public void testConcurrentWriteAndRead()
+ throws InterruptedException, IOException {
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ List<String> words = new ArrayList<>();
+ words.add("ab");
+ words.add("abba");
+ words.add("aba");
+ words.add("bab");
+ words.add("cdd");
+ words.add("efg");
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, words, 20, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, words, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), words.size());
+
+ assertTrue(_resultSet.contains("ab"), "ab not found in result set");
+ assertTrue(_resultSet.contains("abba"), "abba not found in result set");
+ assertTrue(_resultSet.contains("aba"), "aba not found in result set");
+ assertTrue(_resultSet.contains("bab"), "bab not found in result set");
+ assertTrue(_resultSet.contains("cdd"), "cdd not found in result set");
+ assertTrue(_resultSet.contains("efg"), "efg not found in result set");
+ }
+
+ @Test
+ public void testConcurrentWriteWithMultipleThreads()
+ throws InterruptedException, IOException {
+ List<String> firstThreadWords = new ArrayList<>();
+ List<String> secondThreadWords = new ArrayList<>();
+ List<String> mergedThreadWords = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ firstThreadWords.add("ab");
+ firstThreadWords.add("abba");
+ firstThreadWords.add("aba");
+ secondThreadWords.add("bab");
+ secondThreadWords.add("cdd");
+ secondThreadWords.add("efg");
+
+ mergedThreadWords.addAll(firstThreadWords);
+ mergedThreadWords.addAll(secondThreadWords);
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, mergedThreadWords, 20, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, firstThreadWords, 10, countDownLatch);
Review Comment:
I am leaving the test as is -- if nothing else, it demonstrates a capability that is present should we need it someday
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866316222
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeRealTimeTextIndex.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeRealTimeTextIndex implements MutableTextIndex {
Review Comment:
Let's use `NativeMutableTextIndex` instead. Realtime is associated with the table, and within the realtime table we may have both immutable and mutable segment and index. It is slightly confusing to call it realtime index.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r869423892
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java:
##########
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+
+public class NativeAndLuceneMutableTextIndexTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
+ private static final String TEXT_COLUMN_NAME = "testColumnName";
+
+ private RealtimeLuceneTextIndex _realtimeLuceneTextIndex;
+ private NativeMutableTextIndex _nativeMutableTextIndex;
+
+ private List<String> getTextData() {
+ return Arrays.asList("Prince Andrew kept looking with an amused smile from Pierre",
+ "vicomte and from the vicomte to their hostess. In the first moment of",
+ "Pierre’s outburst Anna Pávlovna, despite her social experience, was",
+ "horror-struck. But when she saw that Pierre’s sacrilegious words",
+ "had not exasperated the vicomte, and had convinced herself that it was",
+ "impossible to stop him, she rallied her forces and joined the vicomte in", "a vigorous attack on the orator",
+ "horror-struck. But when she", "she rallied her forces and joined", "outburst Anna Pávlovna",
+ "she rallied her forces and", "despite her social experience", "had not exasperated the vicomte",
+ " despite her social experience", "impossible to stop him", "despite her social experience");
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar");
+ _nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
+ List<String> documents = getTextData();
+
+ for (String doc : documents) {
+ _realtimeLuceneTextIndex.add(doc);
+ _nativeMutableTextIndex.add(doc);
+ }
+
+ SearcherManager searcherManager = _realtimeLuceneTextIndex.getSearcherManager();
+ try {
+ searcherManager.maybeRefresh();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @AfterClass
+ public void tearDown() {
+ _realtimeLuceneTextIndex.close();
+ }
+
+ @Test
+ public void testQueries() {
+ String nativeQuery = "vico.*";
+ String luceneQuery = "vico*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "vicomte";
+ luceneQuery = "vicomte";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "s.*";
+ luceneQuery = "s*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "impossible";
+ luceneQuery = "impossible";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "forc.*s";
+ luceneQuery = "forc*s";
+ testSelectionResults(nativeQuery, luceneQuery);
+ }
+
+ private void testSelectionResults(String nativeQuery, String luceneQuery) {
+ MutableRoaringBitmap resultset = _realtimeLuceneTextIndex.getDocIds(luceneQuery);
+ MutableRoaringBitmap resultset2 = _nativeMutableTextIndex.getDocIds(nativeQuery);
+
Review Comment:
(minor) Remove
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on PR #8636:
URL: https://github.com/apache/pinot/pull/8636#issuecomment-1122597936
> Regards to the Analyser, I looked at the core Lucene class and also did some benchmarking -- due to the fact that only one thread is writing to a single mutable FST at any given point of time, I was not able to see any significant overhead due to the analyser refresh. However, I have attuned the code now per your comments.
We can save one analyser object per document, which has an underlying thread local store and needs to be closed/released after tokenizing the document. If we ingest at high rate, we should be able to see it (at least it is guaranteed to reduce garbage if we reuse it)
> Also, regards to the exceptions, I have been actively trying to limit the exceptions raised to the ones which are significant to the index -- for eg, I do not feel its worth failing the test if a thread is interrupted, or the Lucene SearcherManager failed to refresh. However, again, I have attuned the code now per your comments.
We might want to live with the exception in production code by logging an warning/error and keep the ingestion going, but in the tests we want to catch as much exception as possible as long as they are unexpected so that we can find bugs in the production code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris merged pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris merged PR #8636:
URL: https://github.com/apache/pinot/pull/8636
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866230810
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NativeRealtimeClusterIntegrationTest.java:
##########
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.util.TestUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+
+public class NativeRealtimeClusterIntegrationTest extends BaseClusterIntegrationTest {
Review Comment:
I dont think so, since they will both try to create opposing text indices on the same column.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] yupeng9 commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
yupeng9 commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r865144518
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
whats the CONTAINS semantics that pinot will use? I assume we can not search stopwords with this setting, right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866247898
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
Wow, that's neat. Curious, have you seen a substantial overhead when rebuilding Lucene analysers? We (Lucene community) strives hard to make that an inexpensive operation, so intrigued by your need for this code
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
Wow, that's neat. Curious, have you seen a substantial overhead when rebuilding Lucene analysers? We (Lucene community) strive hard to make that an inexpensive operation, so intrigued by your need for this code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866313273
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
I saw the `Analyzer` implementation stores the stateful value in a thread local object, and I assume the purpose for that is to make it sharable across threads. If the write always come from the same thread, can we reuse the analyzer?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] codecov-commenter commented on pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
codecov-commenter commented on PR #8636:
URL: https://github.com/apache/pinot/pull/8636#issuecomment-1121395275
# [Codecov](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) Report
> Merging [#8636](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) (5c205c3) into [master](https://codecov.io/gh/apache/pinot/commit/7edad89d612438c29591167781518abb1c384519?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) (7edad89) will **decrease** coverage by `56.30%`.
> The diff coverage is `0.00%`.
```diff
@@ Coverage Diff @@
## master #8636 +/- ##
=============================================
- Coverage 70.59% 14.29% -56.31%
+ Complexity 4412 84 -4328
=============================================
Files 1667 1647 -20
Lines 87314 86334 -980
Branches 13085 12986 -99
=============================================
- Hits 61643 12345 -49298
- Misses 21411 73079 +51668
+ Partials 4260 910 -3350
```
| Flag | Coverage Δ | |
|---|---|---|
| integration1 | `?` | |
| integration2 | `?` | |
| unittests1 | `?` | |
| unittests2 | `14.29% <0.00%> (-0.02%)` | :arrow_down: |
Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#carryforward-flags-in-the-pull-request-comment) to find out more.
| [Impacted Files](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) | Coverage Δ | |
|---|---|---|
| [...ava/org/apache/pinot/core/plan/FilterPlanNode.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3QtY29yZS9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29yZS9wbGFuL0ZpbHRlclBsYW5Ob2RlLmphdmE=) | `0.00% <0.00%> (-81.96%)` | :arrow_down: |
| [...local/indexsegment/mutable/MutableSegmentImpl.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc2VnbWVudC1sb2NhbC9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3Qvc2VnbWVudC9sb2NhbC9pbmRleHNlZ21lbnQvbXV0YWJsZS9NdXRhYmxlU2VnbWVudEltcGwuamF2YQ==) | `0.00% <0.00%> (-57.47%)` | :arrow_down: |
| [...ime/impl/invertedindex/NativeMutableTextIndex.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc2VnbWVudC1sb2NhbC9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3Qvc2VnbWVudC9sb2NhbC9yZWFsdGltZS9pbXBsL2ludmVydGVkaW5kZXgvTmF0aXZlTXV0YWJsZVRleHRJbmRleC5qYXZh) | `0.00% <0.00%> (ø)` | |
| [...me/impl/invertedindex/RealtimeLuceneTextIndex.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc2VnbWVudC1sb2NhbC9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3Qvc2VnbWVudC9sb2NhbC9yZWFsdGltZS9pbXBsL2ludmVydGVkaW5kZXgvUmVhbHRpbWVMdWNlbmVUZXh0SW5kZXguamF2YQ==) | `0.00% <ø> (ø)` | |
| [...t/local/segment/creator/impl/text/TermsParser.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc2VnbWVudC1sb2NhbC9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3Qvc2VnbWVudC9sb2NhbC9zZWdtZW50L2NyZWF0b3IvaW1wbC90ZXh0L1Rlcm1zUGFyc2VyLmphdmE=) | `0.00% <0.00%> (ø)` | |
| [...src/main/java/org/apache/pinot/sql/FilterKind.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3QtY29tbW9uL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9waW5vdC9zcWwvRmlsdGVyS2luZC5qYXZh) | `0.00% <0.00%> (-100.00%)` | :arrow_down: |
| [...ain/java/org/apache/pinot/core/data/table/Key.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3QtY29yZS9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29yZS9kYXRhL3RhYmxlL0tleS5qYXZh) | `0.00% <0.00%> (-100.00%)` | :arrow_down: |
| [...in/java/org/apache/pinot/spi/utils/StringUtil.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc3BpL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9waW5vdC9zcGkvdXRpbHMvU3RyaW5nVXRpbC5qYXZh) | `0.00% <0.00%> (-100.00%)` | :arrow_down: |
| [.../java/org/apache/pinot/spi/utils/BooleanUtils.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3Qtc3BpL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9waW5vdC9zcGkvdXRpbHMvQm9vbGVhblV0aWxzLmphdmE=) | `0.00% <0.00%> (-100.00%)` | :arrow_down: |
| [.../java/org/apache/pinot/core/data/table/Record.java](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-cGlub3QtY29yZS9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29yZS9kYXRhL3RhYmxlL1JlY29yZC5qYXZh) | `0.00% <0.00%> (-100.00%)` | :arrow_down: |
| ... and [1351 more](https://codecov.io/gh/apache/pinot/pull/8636/diff?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) | |
------
[Continue to review full report at Codecov](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation). Last update [7edad89...5c205c3](https://codecov.io/gh/apache/pinot/pull/8636?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866175850
##########
pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java:
##########
@@ -246,7 +247,8 @@ private BaseFilterOperator constructPhysicalOperator(FilterContext filter, int n
switch (predicate.getType()) {
case TEXT_CONTAINS:
TextIndexReader textIndexReader = dataSource.getTextIndex();
- if (!(textIndexReader instanceof NativeTextIndexReader)) {
+ if (!(textIndexReader instanceof NativeTextIndexReader)
Review Comment:
Also update the check under `TEXT_MATCH`
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
Is the analyzer stateless? If so, let's create one as `private static final` and reuse it instead of creating one per doc
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
Review Comment:
(minor) Move these 2 non-final values into a separate block for readability
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
+ if (currentDictId == null) {
+
+ currentDictId = _nextDictId;
+ _mutableFST.addPath(token, currentDictId);
+ _termToDictIdMapping.put(token, currentDictId);
+ ++_nextDictId;
+ }
+ addToPostingList(currentDictId);
+ }
+ _nextDocId++;
+ } finally {
+ _writeLock.unlock();
+ }
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDictIds(String searchQuery) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public MutableRoaringBitmap getDocIds(String searchQuery) {
+ MutableRoaringBitmap matchingDocIds = null;
+ try {
+ _readLock.lock();
Review Comment:
(minor) Move this out of the try block
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
+ if (currentDictId == null) {
+
+ currentDictId = _nextDictId;
+ _mutableFST.addPath(token, currentDictId);
+ _termToDictIdMapping.put(token, currentDictId);
+ ++_nextDictId;
+ }
+ addToPostingList(currentDictId);
+ }
+ _nextDocId++;
+ } finally {
+ _writeLock.unlock();
+ }
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDictIds(String searchQuery) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public MutableRoaringBitmap getDocIds(String searchQuery) {
+ MutableRoaringBitmap matchingDocIds = null;
+ try {
+ _readLock.lock();
+ RoaringBitmapWriter<MutableRoaringBitmap> writer = RoaringBitmapWriter.bufferWriter().get();
+ RealTimeRegexpMatcher.regexMatch(searchQuery, _mutableFST, writer::add);
+ ImmutableRoaringBitmap matchingDictIds = writer.get();
+
+ for (PeekableIntIterator it = matchingDictIds.getIntIterator(); it.hasNext(); ) {
+ int dictId = it.next();
+ if (dictId >= 0) {
+ ImmutableRoaringBitmap docIds = _invertedIndex.getDocIds(dictId);
+ if (matchingDocIds == null) {
+ matchingDocIds = docIds.toMutableRoaringBitmap();
+ } else {
+ matchingDocIds.or(docIds);
+ }
+ }
+ }
+ } finally {
+ _readLock.unlock();
+ }
+
+ return matchingDocIds == null ? new MutableRoaringBitmap() : matchingDocIds;
+ }
+
+ @Override
+ public void close()
+ throws IOException {
+ }
+
+ public List<String> analyze(String text, Analyzer analyzer)
+ throws IOException {
+ List<String> result = new ArrayList<>();
+ TokenStream tokenStream = analyzer.tokenStream(_column, text);
+ CharTermAttribute attr = tokenStream.addAttribute(CharTermAttribute.class);
+ tokenStream.reset();
+ while (tokenStream.incrementToken()) {
+ result.add(attr.toString());
+ }
+ return result;
+ }
+
+ /**
+ * Adds the given value to the posting list.
+ */
+ void addToPostingList(int dictId) {
Review Comment:
(minor) Inline this
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
Review Comment:
Use `computeIfAbsent` to avoid the extra map lookup
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
+ if (currentDictId == null) {
+
+ currentDictId = _nextDictId;
+ _mutableFST.addPath(token, currentDictId);
+ _termToDictIdMapping.put(token, currentDictId);
+ ++_nextDictId;
+ }
+ addToPostingList(currentDictId);
+ }
+ _nextDocId++;
+ } finally {
+ _writeLock.unlock();
+ }
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDictIds(String searchQuery) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public MutableRoaringBitmap getDocIds(String searchQuery) {
+ MutableRoaringBitmap matchingDocIds = null;
+ try {
+ _readLock.lock();
+ RoaringBitmapWriter<MutableRoaringBitmap> writer = RoaringBitmapWriter.bufferWriter().get();
+ RealTimeRegexpMatcher.regexMatch(searchQuery, _mutableFST, writer::add);
+ ImmutableRoaringBitmap matchingDictIds = writer.get();
+
+ for (PeekableIntIterator it = matchingDictIds.getIntIterator(); it.hasNext(); ) {
+ int dictId = it.next();
+ if (dictId >= 0) {
+ ImmutableRoaringBitmap docIds = _invertedIndex.getDocIds(dictId);
+ if (matchingDocIds == null) {
+ matchingDocIds = docIds.toMutableRoaringBitmap();
+ } else {
+ matchingDocIds.or(docIds);
+ }
+ }
+ }
+ } finally {
+ _readLock.unlock();
+ }
+
+ return matchingDocIds == null ? new MutableRoaringBitmap() : matchingDocIds;
Review Comment:
No need to create a bitmap for the dict ids. This can be simplified
```suggestion
MutableRoaringBitmap matchingDocIds = new MutableRoaringBitmap();
_readLock.lock();
try {
RealTimeRegexpMatcher.regexMatch(searchQuery, _mutableFST, dictId -> matchingDocIds.or(_invertedIndex.getDocIds(dictId)));
return matchingDocIds;
}
} finally {
_readLock.unlock();
}
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
Review Comment:
Keep the underlying stack trace for debugging purpose
```suggestion
throw new RuntimeException(e);
```
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NativeRealtimeClusterIntegrationTest.java:
##########
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.util.TestUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+
+public class NativeRealtimeClusterIntegrationTest extends BaseClusterIntegrationTest {
Review Comment:
Can we refactor `LuceneRealtimeClusterIntegrationTest` to `TextIndexRealtimeClusterIntegrationTest` and integrate this into the same test?
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
+ if (currentDictId == null) {
+
Review Comment:
(nit) remove empty line
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
Review Comment:
Ideally we should use a mutable dictionary to store the value mapping so that it can be off-heap. We may add a TODO here and address that separately.
One low hanging optimization is to use `Object2IntOpenHashMap` which has better performance and lower memory footprint
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
Review Comment:
Suggest renaming it to `NativeMutableTextIndex`
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.get(token);
+ if (currentDictId == null) {
+
+ currentDictId = _nextDictId;
Review Comment:
(nit) Can be simplified
```suggestion
currentDictId = _nextDictId++;
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ try {
+ _writeLock.lock();
Review Comment:
(minor) Move this out of the `try` block
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r867751836
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndex.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeMutableTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeMutableTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
I dont think so -- since StandardAnalyzer uses a parser that is JavaCC based. I tried the same and ran into issues.
However, the snippet @kkrugler shared above has legs. I reused the same (refactored it a bit) and it gives the required results.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866314846
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NativeRealtimeClusterIntegrationTest.java:
##########
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.util.TestUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+
+public class NativeRealtimeClusterIntegrationTest extends BaseClusterIntegrationTest {
Review Comment:
The test data is created in the test, and you can easily add the same data to two columns, or use derived column
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866316849
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeRealTimeTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
Review Comment:
Please import the one from `org.testng.Assert.*`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866231679
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ private final Map<String, Integer> _termToDictIdMapping;
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ public RealtimeNativeTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new HashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
No, Lucene analysers need to be refreshed every time they are used since they maintain state internally
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r866231420
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndex.java:
##########
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.RoaringBitmapWriter;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class RealtimeNativeTextIndex implements MutableTextIndex {
Review Comment:
Renamed to NativeRealTimeTextIndex, to match with the Lucene version of it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] atris commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
atris commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r865234082
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeNativeTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+
+public class RealtimeNativeTextIndexConcurrentTest {
+ private ExecutorService _threadPool;
+ private Set<String> _resultSet;
+
+ @BeforeClass
+ private void setup() {
+ _threadPool = Executors.newFixedThreadPool(10);
+ _resultSet = new ConcurrentSkipListSet<>();
+ }
+
+ @AfterClass
+ private void shutDown() {
+ _threadPool.shutdown();
+ }
+
+ @Test
+ public void testConcurrentWriteAndRead()
+ throws InterruptedException, IOException {
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ List<String> words = new ArrayList<>();
+ words.add("ab");
+ words.add("abba");
+ words.add("aba");
+ words.add("bab");
+ words.add("cdd");
+ words.add("efg");
+
+ try (RealtimeNativeTextIndex textIndex = new RealtimeNativeTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, words, 10, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, words, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), words.size());
+
+ assertTrue("ab not found in result set", _resultSet.contains("ab"));
+ assertTrue("abba not found in result set", _resultSet.contains("abba"));
+ assertTrue("aba not found in result set", _resultSet.contains("aba"));
+ assertTrue("bab not found in result set", _resultSet.contains("bab"));
+ assertTrue("cdd not found in result set", _resultSet.contains("cdd"));
+ assertTrue("efg not found in result set", _resultSet.contains("efg"));
+ }
+
+ @Test
+ public void testConcurrentWriteWithMultipleThreads()
+ throws InterruptedException, IOException {
+ List<String> firstThreadWords = new ArrayList<>();
+ List<String> secondThreadWords = new ArrayList<>();
+ List<String> mergedThreadWords = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ firstThreadWords.add("ab");
+ firstThreadWords.add("abba");
+ firstThreadWords.add("aba");
+ secondThreadWords.add("bab");
+ secondThreadWords.add("cdd");
+ secondThreadWords.add("efg");
+
+ mergedThreadWords.addAll(firstThreadWords);
+ mergedThreadWords.addAll(secondThreadWords);
+
+ try (RealtimeNativeTextIndex textIndex = new RealtimeNativeTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, mergedThreadWords, 10, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, firstThreadWords, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, secondThreadWords, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), mergedThreadWords.size());
+
+ assertTrue("ab not found in result set", _resultSet.contains("ab"));
Review Comment:
That's the message that will be shown if the element is not found
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #8636: Introduce Real Time Native Text Index
Posted by GitBox <gi...@apache.org>.
Jackie-Jiang commented on code in PR #8636:
URL: https://github.com/apache/pinot/pull/8636#discussion_r868468538
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
+ private static final String TEXT_COLUMN_NAME = "testColumnName";
+
+ private RealtimeLuceneTextIndex _realtimeLuceneTextIndex;
+ private NativeMutableTextIndex _nativeMutableTextIndex;
+
+ private List<String> getTextData() {
+ return Arrays.asList("Prince Andrew kept looking with an amused smile from Pierre",
+ "vicomte and from the vicomte to their hostess. In the first moment of",
+ "Pierre’s outburst Anna Pávlovna, despite her social experience, was",
+ "horror-struck. But when she saw that Pierre’s sacrilegious words",
+ "had not exasperated the vicomte, and had convinced herself that it was",
+ "impossible to stop him, she rallied her forces and joined the vicomte in", "a vigorous attack on the orator",
+ "horror-struck. But when she", "she rallied her forces and joined", "outburst Anna Pávlovna",
+ "she rallied her forces and", "despite her social experience", "had not exasperated the vicomte",
+ " despite her social experience", "impossible to stop him", "despite her social experience");
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar");
+ _nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
+ List<String> documents = getTextData();
+
+ for (String doc : documents) {
+ _realtimeLuceneTextIndex.add(doc);
+ _nativeMutableTextIndex.add(doc);
+ }
+
+ SearcherManager searcherManager = _realtimeLuceneTextIndex.getSearcherManager();
+ try {
+ searcherManager.maybeRefresh();
+ } catch (Exception e) {
+ // we should never be here since the locking semantics between MutableSegmentImpl::destroy()
Review Comment:
This is a test, and we do want to catch the unexpected exception. Let's remove the try-catch block and let the test fail if it happens
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TextIndicesRealtimeClusterIntegrationTest.java:
##########
@@ -201,16 +219,35 @@ public void testTextSearchCountQuery()
//Lucene index on consuming segments to update the latest records
TestUtils.waitForCondition(aVoid -> {
try {
- return getTextColumnQueryResult() == NUM_MATCHING_RECORDS;
+ return getTextColumnQueryResult(TEST_TEXT_COLUMN_QUERY) == NUM_MATCHING_RECORDS;
} catch (Exception e) {
fail("Caught exception while getting text column query result");
return false;
}
}, 10_000L, "Failed to reach expected number of matching records");
}
- private long getTextColumnQueryResult()
+ @Test
+ public void testTextSearchCountQueryNative()
+ throws Exception {
+ // Keep posting queries until all records are consumed
+ long previousResult = 0;
+ while (getCurrentCountStarResult() < NUM_RECORDS) {
+ long result = getTextColumnQueryResult(TEST_TEXT_COLUMN_QUERY_NATIVE);
+ assertTrue(result >= previousResult);
+ previousResult = result;
+ Thread.sleep(100);
+ }
+
+ try {
+ Assert.assertTrue(getTextColumnQueryResult(TEST_TEXT_COLUMN_QUERY_NATIVE) == NUM_MATCHING_RECORDS_NATIVE);
+ } catch (Exception e) {
+ fail("Caught exception while getting text column query result");
+ }
Review Comment:
No need to catch the exception
```suggestion
assertTrue(getTextColumnQueryResult(TEST_TEXT_COLUMN_QUERY_NATIVE) == NUM_MATCHING_RECORDS_NATIVE);
```
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
+ private static final String TEXT_COLUMN_NAME = "testColumnName";
+
+ private RealtimeLuceneTextIndex _realtimeLuceneTextIndex;
+ private NativeMutableTextIndex _nativeMutableTextIndex;
+
+ private List<String> getTextData() {
+ return Arrays.asList("Prince Andrew kept looking with an amused smile from Pierre",
+ "vicomte and from the vicomte to their hostess. In the first moment of",
+ "Pierre’s outburst Anna Pávlovna, despite her social experience, was",
+ "horror-struck. But when she saw that Pierre’s sacrilegious words",
+ "had not exasperated the vicomte, and had convinced herself that it was",
+ "impossible to stop him, she rallied her forces and joined the vicomte in", "a vigorous attack on the orator",
+ "horror-struck. But when she", "she rallied her forces and joined", "outburst Anna Pávlovna",
+ "she rallied her forces and", "despite her social experience", "had not exasperated the vicomte",
+ " despite her social experience", "impossible to stop him", "despite her social experience");
+ }
+
+ @BeforeClass
+ public void setUp()
Review Comment:
Add a `tearDown()` method to delete the `INDEX_DIR`
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndex.java:
##########
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.segment.creator.impl.text.TermsParser;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeMutableTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+ private final TermsParser _termsParser;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeMutableTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ _termsParser = new TermsParser(new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET), _column);
+ }
+
+ @Override
+ public void add(String document) {
+ Iterable<String> tokens;
+
+ tokens = _termsParser.parse(document);
Review Comment:
I tried this block of code and it works
```
List<String> tokens = new ArrayList<>();
try (TokenStream tokenStream = _analyzer.tokenStream(_column, document)) {
tokenStream.reset();
CharTermAttribute attribute = tokenStream.getAttribute(CharTermAttribute.class);
while (tokenStream.incrementToken()) {
tokens.add(attribute.toString());
}
tokenStream.end();
} catch (IOException e) {
throw new RuntimeException("Caught exception while tokenizing the document for column: " + _column, e);
}
```
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
+ private static final String TEXT_COLUMN_NAME = "testColumnName";
+
+ private RealtimeLuceneTextIndex _realtimeLuceneTextIndex;
+ private NativeMutableTextIndex _nativeMutableTextIndex;
+
+ private List<String> getTextData() {
+ return Arrays.asList("Prince Andrew kept looking with an amused smile from Pierre",
+ "vicomte and from the vicomte to their hostess. In the first moment of",
+ "Pierre’s outburst Anna Pávlovna, despite her social experience, was",
+ "horror-struck. But when she saw that Pierre’s sacrilegious words",
+ "had not exasperated the vicomte, and had convinced herself that it was",
+ "impossible to stop him, she rallied her forces and joined the vicomte in", "a vigorous attack on the orator",
+ "horror-struck. But when she", "she rallied her forces and joined", "outburst Anna Pávlovna",
+ "she rallied her forces and", "despite her social experience", "had not exasperated the vicomte",
+ " despite her social experience", "impossible to stop him", "despite her social experience");
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar");
+ _nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
+ List<String> documents = getTextData();
+
+ for (String doc : documents) {
+ _realtimeLuceneTextIndex.add(doc);
+ _nativeMutableTextIndex.add(doc);
+ }
+
+ SearcherManager searcherManager = _realtimeLuceneTextIndex.getSearcherManager();
+ try {
+ searcherManager.maybeRefresh();
+ } catch (Exception e) {
+ // we should never be here since the locking semantics between MutableSegmentImpl::destroy()
+ // and this code along with volatile state "isSegmentDestroyed" protect against the cases
+ // where this thread might attempt to refresh a realtime lucene reader after it has already
+ // been closed duing segment destroy.
+ }
+ }
+
+ @Override
+ protected String getFilter() {
+ return null;
+ }
+
+ @Override
+ protected IndexSegment getIndexSegment() {
+ return null;
+ }
+
+ @Override
+ protected List<IndexSegment> getIndexSegments() {
+ return null;
+ }
+
+ @Test
+ public void testQueries() {
+ String nativeQuery = "vico.*";
+ String luceneQuery = "vico*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "vicomte";
+ luceneQuery = "vicomte";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "s.*";
+ luceneQuery = "s*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "impossible";
+ luceneQuery = "impossible";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "forc.*s";
+ luceneQuery = "forc*s";
+ testSelectionResults(nativeQuery, luceneQuery);
+ }
+
+ private void testSelectionResults(String nativeQuery, String luceneQuery) {
+ MutableRoaringBitmap resultset = _realtimeLuceneTextIndex.getDocIds(luceneQuery);
+ Assert.assertNotNull(resultset);
Review Comment:
You can directly compare the bitmaps
```suggestion
assertEquals(_nativeMutableTextIndex.getDocIds(nativeQuery), _realtimeLuceneTextIndex.getDocIds(luceneQuery));
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndex.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeMutableTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeMutableTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ }
+
+ @Override
+ public void add(String document) {
+ List<String> tokens;
+ try {
+ tokens = analyze(document, new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET));
Review Comment:
I tried reusing the analyzer and am able to do that without causing issues. Here are the code I tried:
```
List<String> tokens = new ArrayList<>();
try (TokenStream tokenStream = _analyzer.tokenStream(_column, document)) {
tokenStream.reset();
CharTermAttribute attribute = tokenStream.getAttribute(CharTermAttribute.class);
while (tokenStream.incrementToken()) {
tokens.add(attribute.toString());
}
tokenStream.end();
} catch (IOException e) {
throw new RuntimeException("Caught exception while tokenizing the document for column: " + _column, e);
}
```
If I remove the `end()` and `close()` for the `TokenStream`, it will hang, and I think that's why you run into issue for your previous version of the code
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndex.java:
##########
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.IOException;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator;
+import org.apache.pinot.segment.local.segment.creator.impl.text.TermsParser;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFST;
+import org.apache.pinot.segment.local.utils.nativefst.mutablefst.MutableFSTImpl;
+import org.apache.pinot.segment.local.utils.nativefst.utils.RealTimeRegexpMatcher;
+import org.apache.pinot.segment.spi.index.mutable.MutableTextIndex;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class NativeMutableTextIndex implements MutableTextIndex {
+ private final String _column;
+ private final MutableFST _mutableFST;
+ private final RealtimeInvertedIndex _invertedIndex;
+ //TODO: Move to mutable dictionary
+ private final Object2IntOpenHashMap<String> _termToDictIdMapping;
+ private final ReentrantReadWriteLock.ReadLock _readLock;
+ private final ReentrantReadWriteLock.WriteLock _writeLock;
+ private final TermsParser _termsParser;
+
+ private int _nextDocId = 0;
+ private int _nextDictId = 0;
+
+ public NativeMutableTextIndex(String column) {
+ _column = column;
+ _mutableFST = new MutableFSTImpl();
+ _termToDictIdMapping = new Object2IntOpenHashMap<>();
+ _invertedIndex = new RealtimeInvertedIndex();
+
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ _readLock = readWriteLock.readLock();
+ _writeLock = readWriteLock.writeLock();
+ _termsParser = new TermsParser(new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET), _column);
+ }
+
+ @Override
+ public void add(String document) {
+ Iterable<String> tokens;
+
+ tokens = _termsParser.parse(document);
+ _writeLock.lock();
+ try {
+ for (String token : tokens) {
+ Integer currentDictId = _termToDictIdMapping.computeIntIfAbsent(token, k -> {
+ int localDictId = _nextDictId++;
+ _mutableFST.addPath(token, localDictId);
+ return localDictId;
+ });
+ _invertedIndex.add(currentDictId, _nextDocId);
+ }
+ _nextDocId++;
+ } finally {
+ _writeLock.unlock();
+ }
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDictIds(String searchQuery) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public MutableRoaringBitmap getDocIds(String searchQuery) {
+ MutableRoaringBitmap matchingDocIds = new MutableRoaringBitmap();
+ _readLock.lock();
+ try {
+ RealTimeRegexpMatcher.regexMatch(searchQuery, _mutableFST,
+ dictId -> matchingDocIds.or(_invertedIndex.getDocIds(dictId)));
+ return matchingDocIds;
+ } finally {
+ _readLock.unlock();
+ }
+ }
+
+ @Override
+ public void close()
+ throws IOException {
Review Comment:
We should close the analyzer
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
Review Comment:
Don't extend `BaseQueriesTest` because it is not a queries test. Suggest renaming it to `NativeAndLuceneMutableTextIndexTest` (this is not only testing FST either) and move it under the package `org.apache.pinot.segment.local.realtime.impl.invertedindex`
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeMutableTextIndexConcurrentTest.java:
##########
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.realtime.impl.invertedindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+public class NativeMutableTextIndexConcurrentTest {
+ private ExecutorService _threadPool;
+ private Set<String> _resultSet;
+
+ @BeforeClass
+ private void setup() {
+ _threadPool = Executors.newFixedThreadPool(10);
+ _resultSet = new ConcurrentSkipListSet<>();
+ }
+
+ @AfterClass
+ private void shutDown() {
+ _threadPool.shutdown();
+ }
+
+ @Test
+ public void testConcurrentWriteAndRead()
+ throws InterruptedException, IOException {
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ List<String> words = new ArrayList<>();
+ words.add("ab");
+ words.add("abba");
+ words.add("aba");
+ words.add("bab");
+ words.add("cdd");
+ words.add("efg");
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, words, 20, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, words, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ countDownLatch.await();
+ }
+
+ assertEquals(_resultSet.size(), words.size());
+
+ assertTrue(_resultSet.contains("ab"), "ab not found in result set");
+ assertTrue(_resultSet.contains("abba"), "abba not found in result set");
+ assertTrue(_resultSet.contains("aba"), "aba not found in result set");
+ assertTrue(_resultSet.contains("bab"), "bab not found in result set");
+ assertTrue(_resultSet.contains("cdd"), "cdd not found in result set");
+ assertTrue(_resultSet.contains("efg"), "efg not found in result set");
+ }
+
+ @Test
+ public void testConcurrentWriteWithMultipleThreads()
+ throws InterruptedException, IOException {
+ List<String> firstThreadWords = new ArrayList<>();
+ List<String> secondThreadWords = new ArrayList<>();
+ List<String> mergedThreadWords = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ firstThreadWords.add("ab");
+ firstThreadWords.add("abba");
+ firstThreadWords.add("aba");
+ secondThreadWords.add("bab");
+ secondThreadWords.add("cdd");
+ secondThreadWords.add("efg");
+
+ mergedThreadWords.addAll(firstThreadWords);
+ mergedThreadWords.addAll(secondThreadWords);
+
+ try (NativeMutableTextIndex textIndex = new NativeMutableTextIndex("testFSTColumn")) {
+ _threadPool.submit(() -> {
+ try {
+ performReads(textIndex, mergedThreadWords, 20, 200, countDownLatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ _threadPool.submit(() -> {
+ try {
+ performWrites(textIndex, firstThreadWords, 10, countDownLatch);
Review Comment:
We don't need to test multiple writer scenario (although it seems work). For mutable index, pinot always follow single writer multiple readers. This gives us a lot of optimization opportunities since we don't need to worry about concurrent modification of the index
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
+ private static final String TEXT_COLUMN_NAME = "testColumnName";
+
+ private RealtimeLuceneTextIndex _realtimeLuceneTextIndex;
+ private NativeMutableTextIndex _nativeMutableTextIndex;
+
+ private List<String> getTextData() {
+ return Arrays.asList("Prince Andrew kept looking with an amused smile from Pierre",
+ "vicomte and from the vicomte to their hostess. In the first moment of",
+ "Pierre’s outburst Anna Pávlovna, despite her social experience, was",
+ "horror-struck. But when she saw that Pierre’s sacrilegious words",
+ "had not exasperated the vicomte, and had convinced herself that it was",
+ "impossible to stop him, she rallied her forces and joined the vicomte in", "a vigorous attack on the orator",
+ "horror-struck. But when she", "she rallied her forces and joined", "outburst Anna Pávlovna",
+ "she rallied her forces and", "despite her social experience", "had not exasperated the vicomte",
+ " despite her social experience", "impossible to stop him", "despite her social experience");
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar");
+ _nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
+ List<String> documents = getTextData();
+
+ for (String doc : documents) {
+ _realtimeLuceneTextIndex.add(doc);
+ _nativeMutableTextIndex.add(doc);
+ }
+
+ SearcherManager searcherManager = _realtimeLuceneTextIndex.getSearcherManager();
+ try {
+ searcherManager.maybeRefresh();
+ } catch (Exception e) {
+ // we should never be here since the locking semantics between MutableSegmentImpl::destroy()
+ // and this code along with volatile state "isSegmentDestroyed" protect against the cases
+ // where this thread might attempt to refresh a realtime lucene reader after it has already
+ // been closed duing segment destroy.
+ }
+ }
+
+ @Override
+ protected String getFilter() {
+ return null;
+ }
+
+ @Override
+ protected IndexSegment getIndexSegment() {
+ return null;
+ }
+
+ @Override
+ protected List<IndexSegment> getIndexSegments() {
+ return null;
+ }
+
+ @Test
+ public void testQueries() {
+ String nativeQuery = "vico.*";
+ String luceneQuery = "vico*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "vicomte";
+ luceneQuery = "vicomte";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "s.*";
+ luceneQuery = "s*";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "impossible";
+ luceneQuery = "impossible";
+ testSelectionResults(nativeQuery, luceneQuery);
+
+ nativeQuery = "forc.*s";
+ luceneQuery = "forc*s";
+ testSelectionResults(nativeQuery, luceneQuery);
+ }
+
+ private void testSelectionResults(String nativeQuery, String luceneQuery) {
+ MutableRoaringBitmap resultset = _realtimeLuceneTextIndex.getDocIds(luceneQuery);
+ Assert.assertNotNull(resultset);
Review Comment:
(nit) Use static import for asserts in test
##########
pinot-core/src/test/java/org/apache/pinot/queries/LuceneFSTVsNativeMutableFSTTest.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.queries;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex;
+import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex;
+import org.apache.pinot.segment.spi.IndexSegment;
+import org.roaringbitmap.PeekableIntIterator;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class LuceneFSTVsNativeMutableFSTTest extends BaseQueriesTest {
+ private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RealTimeNativeVsLuceneTest");
Review Comment:
(minor) Change the file name to match the test name
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/mutablefst/MutableFSTConcurrentTest.java:
##########
@@ -102,8 +100,54 @@ public void testConcurrentWriteAndRead()
assertTrue("efg not found in result set", _resultSet.contains("efg"));
}
- private void performReads(MutableFST fst, List<String> words, int count,
- long sleepTime)
+ @Test
+ public void testConcurrentLongWriteAndRead()
+ throws InterruptedException {
+ MutableFST mutableFST = new MutableFSTImpl();
+ List<String> words = new ArrayList<>();
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+
+ mutableFST.addPath("ab", 1);
+
+ List<Pair<String, Integer>> wordsWithMetadata = new ArrayList<>();
+
+ // Add some write pressure
+ wordsWithMetadata.add(Pair.of("egegdgrbsbrsegzgzegzegegjntnmtj", 2));
+ wordsWithMetadata.add(Pair.of("hrwbwefweg4wreghrtbrassregfesfefefefzew4ere", 2));
+ wordsWithMetadata.add(Pair.of("easzegfegrertegbxzzez3erfezgzeddzdewstfefed", 2));
+ wordsWithMetadata.add(Pair.of("tjntrhndsrsgezgrsxzetgteszetgezfzezedrefzdzdzdzdz", 2));
+ wordsWithMetadata.add(Pair.of("abacxcvbnmlkjjhgfsaqwertyuioopzxcvbnmllkjshfgsfawieeiuefgeurfeoafa", 2));
+
+ words.add("abacxcvbnmlkjjhgfsaqwertyuioopzxcvbnmllkjshfgsfawieeiuefgeurfeoafa");
+
+ _threadPool.submit(() -> {
+ try {
+ performReads(mutableFST, words, 10, 10, countDownLatch);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
Review Comment:
Please don't swallow the exception in the test. We want to catch them
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org