You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/06/07 20:04:47 UTC
[pinot] branch master updated: compress nullvector bitmap upon seal (#10852)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0f58e48c8e compress nullvector bitmap upon seal (#10852)
0f58e48c8e is described below
commit 0f58e48c8e82739cfffc760fcea65c8165028e9d
Author: Xiaobing <61...@users.noreply.github.com>
AuthorDate: Wed Jun 7 13:04:40 2023 -0700
compress nullvector bitmap upon seal (#10852)
---
.../creator/impl/nullvalue/NullValueVectorCreator.java | 18 ++++++++++--------
.../impl/nullvalue/NullValueVectorCreatorTest.java | 4 ++--
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
index 2627248749..2055087909 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
@@ -27,8 +27,8 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.IndexCreator;
-import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
+import org.roaringbitmap.RoaringBitmap;
+import org.roaringbitmap.RoaringBitmapWriter;
/**
@@ -42,7 +42,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
* null, which is true for all index creators types unless this one.
*/
public class NullValueVectorCreator implements IndexCreator {
- private final MutableRoaringBitmap _nullBitmap = new MutableRoaringBitmap();
+ private final RoaringBitmapWriter<RoaringBitmap> _bitmapWriter;
private final File _nullValueVectorFile;
@Override
@@ -58,26 +58,28 @@ public class NullValueVectorCreator implements IndexCreator {
}
public NullValueVectorCreator(File indexDir, String columnName) {
+ _bitmapWriter = RoaringBitmapWriter.writer().get();
_nullValueVectorFile = new File(indexDir, columnName + V1Constants.Indexes.NULLVALUE_VECTOR_FILE_EXTENSION);
}
public void setNull(int docId) {
- _nullBitmap.add(docId);
+ _bitmapWriter.add(docId);
}
public void seal()
throws IOException {
// Create null value vector file only if the bitmap is not empty
- if (!_nullBitmap.isEmpty()) {
+ RoaringBitmap nullBitmap = _bitmapWriter.get();
+ if (!nullBitmap.isEmpty()) {
try (DataOutputStream outputStream = new DataOutputStream(new FileOutputStream(_nullValueVectorFile))) {
- _nullBitmap.serialize(outputStream);
+ nullBitmap.serialize(outputStream);
}
}
}
@VisibleForTesting
- ImmutableRoaringBitmap getNullBitmap() {
- return _nullBitmap;
+ RoaringBitmap getNullBitmap() {
+ return _bitmapWriter.get();
}
@Override
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreatorTest.java
index 1099ec0000..8bb530bb8b 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreatorTest.java
@@ -21,7 +21,7 @@ package org.apache.pinot.segment.local.segment.creator.impl.nullvalue;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -49,7 +49,7 @@ public class NullValueVectorCreatorTest {
for (int i = 0; i < 100; i++) {
creator.setNull(i);
}
- ImmutableRoaringBitmap nullBitmap = creator.getNullBitmap();
+ RoaringBitmap nullBitmap = creator.getNullBitmap();
for (int i = 0; i < 100; i++) {
Assert.assertTrue(nullBitmap.contains(i));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org