You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/04/02 18:14:08 UTC
[incubator-pinot] branch fix_realtime_race_condition updated:
Handle multi-valued column
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch fix_realtime_race_condition
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/fix_realtime_race_condition by this push:
new 0a914da Handle multi-valued column
0a914da is described below
commit 0a914da71214b19f36c55fe25f9ca8d460d1c4cc
Author: Jackie (Xiaotian) Jiang <xa...@linkedin.com>
AuthorDate: Tue Apr 2 11:13:54 2019 -0700
Handle multi-valued column
---
.../invertedindex/RealtimeInvertedIndexReader.java | 12 ++++--
.../RealtimeInvertedIndexReaderTest.java | 46 +++++++++++++++++-----
2 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
index 1250da0..28db4a1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
@@ -25,6 +25,10 @@ import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
+/**
+ * Real-time bitmap based inverted index reader which allows adding values on the fly.
+ * <p>This class is thread-safe for single writer multiple readers.
+ */
public class RealtimeInvertedIndexReader implements InvertedIndexReader<MutableRoaringBitmap> {
private final List<ThreadSafeMutableRoaringBitmap> _bitmaps = new ArrayList<>();
private final ReentrantReadWriteLock.ReadLock _readLock;
@@ -37,7 +41,7 @@ public class RealtimeInvertedIndexReader implements InvertedIndexReader<MutableR
}
/**
- * Add the document id to the bitmap for the given dictionary id.
+ * Adds the doc Id to the given dict Id's bitmap.
*/
public void add(int dictId, int docId) {
if (_bitmaps.size() == dictId) {
@@ -61,10 +65,10 @@ public class RealtimeInvertedIndexReader implements InvertedIndexReader<MutableR
try {
_readLock.lock();
// NOTE: the given dict Id might not be added to the inverted index yet. We first add the value to the dictionary.
- // Before the value is added to the inverted index, the query might have a predicate which matches the newly added
+ // Before the value is added to the inverted index, the query might have predicates that match the newly added
// value. In that case, the given dictionary Id does not exist in the inverted index, and we return an empty
- // bitmap.
- if (_bitmaps.size() == dictId) {
+ // bitmap. For multi-valued column, the dict Id might be larger than the bitmap size (not equal).
+ if (_bitmaps.size() <= dictId) {
return new MutableRoaringBitmap();
}
bitmap = _bitmaps.get(dictId);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
index 2191a23..7ecdc56 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
@@ -32,47 +32,75 @@ public class RealtimeInvertedIndexReaderTest {
public void testRealtimeInvertedIndexReader() {
RealtimeInvertedIndexReader realtimeInvertedIndexReader = new RealtimeInvertedIndexReader();
- // Dict Id not added yet
+ // Add dict Id 0, doc Id 0 to the inverted index (single-value dict Id not added yet)
+ // Before adding
MutableRoaringBitmap docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 0, doc Id 0 to the inverted index
+ // After adding
realtimeInvertedIndexReader.add(0, 0);
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertFalse(docIds.contains(1));
+
+ // Add dict Id 0, doc Id 1 to the inverted index (single-value dict Id already added)
+ // Before adding
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 0, doc Id 1 to the inverted index
realtimeInvertedIndexReader.add(0, 1);
+ // After adding
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertTrue(docIds.contains(1));
+
+ // Add dict Id 1 and 2, doc Id 2 to the inverted index (multi-value dict Ids not added yet)
+ // Before adding dict Id 1
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 1, doc Id 1 to the inverted index
- realtimeInvertedIndexReader.add(1, 1);
+ docIds = realtimeInvertedIndexReader.getDocIds(2);
+ assertNotNull(docIds);
+ assertTrue(docIds.isEmpty());
+ // After adding dict Id 1 but before adding dict Id 2
+ realtimeInvertedIndexReader.add(1, 2);
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(2));
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertFalse(docIds.contains(0));
- assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
docIds = realtimeInvertedIndexReader.getDocIds(2);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
+ // After adding dict Id 2
+ realtimeInvertedIndexReader.add(2, 2);
+ docIds = realtimeInvertedIndexReader.getDocIds(0);
+ assertNotNull(docIds);
+ assertFalse(docIds.isEmpty());
+ assertTrue(docIds.contains(0));
+ assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(2));
+ docIds = realtimeInvertedIndexReader.getDocIds(1);
+ assertNotNull(docIds);
+ assertFalse(docIds.isEmpty());
+ assertFalse(docIds.contains(0));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
+ docIds = realtimeInvertedIndexReader.getDocIds(2);
+ assertFalse(docIds.isEmpty());
+ assertFalse(docIds.contains(0));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org