You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/03/07 02:16:41 UTC
[pinot] 01/01: Example change in SVScanDocIdIterator for PR #10372
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch example_10372
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 345cbc6c5f27a5117e974c57413335e5cd3b3eea
Author: Xiaotian (Jackie) Jiang <ja...@gmail.com>
AuthorDate: Mon Mar 6 18:16:24 2023 -0800
Example change in SVScanDocIdIterator for PR #10372
---
.../dociditerators/SVScanDocIdIterator.java | 49 +++++++++++++---------
1 file changed, 30 insertions(+), 19 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
index 1a4f9c2c54..e910c3bf59 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.dociditerators;
+import com.google.common.annotations.VisibleForTesting;
import javax.annotation.Nullable;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.segment.spi.Constants;
@@ -44,16 +45,17 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private final ForwardIndexReaderContext _readerContext;
private final int _numDocs;
private final ValueMatcher _valueMatcher;
- private final int[] _batch = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
- private int _firstMismatch;
- private int _cursor;
private final int _cardinality;
+ private final int _batchSize;
+ private final int[] _batch;
+ private int _firstMismatch;
+ private int _cursor;
private int _nextDocId = 0;
private long _numEntriesScanned = 0L;
public SVScanDocIdIterator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int numDocs,
- @Nullable NullValueVectorReader nullValueReader) {
+ @Nullable NullValueVectorReader nullValueReader, int batchSize) {
_predicateEvaluator = predicateEvaluator;
_reader = dataSource.getForwardIndex();
_readerContext = _reader.createContext();
@@ -64,9 +66,16 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
}
_valueMatcher = getValueMatcher(nullBitmap);
_cardinality = dataSource.getDataSourceMetadata().getCardinality();
+ _batchSize = batchSize;
+ _batch = new int[batchSize];
+ }
+
+ public SVScanDocIdIterator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int numDocs,
+ @Nullable NullValueVectorReader nullValueReader) {
+ this(predicateEvaluator, dataSource, numDocs, nullValueReader, OPTIMAL_ITERATOR_BATCH_SIZE);
}
- // for testing
+ @VisibleForTesting
public SVScanDocIdIterator(PredicateEvaluator predicateEvaluator, ForwardIndexReader reader, int numDocs,
@Nullable NullValueVectorReader nullValueReader) {
_predicateEvaluator = predicateEvaluator;
@@ -79,6 +88,8 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
}
_valueMatcher = getValueMatcher(nullBitmap);
_cardinality = -1;
+ _batchSize = OPTIMAL_ITERATOR_BATCH_SIZE;
+ _batch = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
}
@Override
@@ -87,7 +98,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
int limit;
int batchSize = 0;
do {
- limit = Math.min(_numDocs - _nextDocId, OPTIMAL_ITERATOR_BATCH_SIZE);
+ limit = Math.min(_numDocs - _nextDocId, _batchSize);
if (limit > 0) {
for (int i = 0; i < limit; i++) {
_batch[i] = _nextDocId + i;
@@ -125,10 +136,10 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
if (docIds.isEmpty()) {
return new MutableRoaringBitmap();
}
- RoaringBitmapWriter<MutableRoaringBitmap> result = RoaringBitmapWriter.bufferWriter()
- .expectedRange(docIds.first(), docIds.last()).runCompress(false).get();
+ RoaringBitmapWriter<MutableRoaringBitmap> result =
+ RoaringBitmapWriter.bufferWriter().expectedRange(docIds.first(), docIds.last()).runCompress(false).get();
BatchIterator docIdIterator = docIds.getBatchIterator();
- int[] buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ int[] buffer = new int[_batchSize];
while (docIdIterator.hasNext()) {
int limit = docIdIterator.nextBatch(buffer);
if (limit > 0) {
@@ -264,7 +275,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class DictIdMatcher implements ValueMatcher {
- private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final int[] _buffer = new int[_batchSize];
@Override
public boolean doesValueMatch(int docId) {
@@ -280,7 +291,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class DictIdMatcherAndNullHandler implements ValueMatcher {
- private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final int[] _buffer = new int[_batchSize];
private final ImmutableRoaringBitmap _nullBitmap;
public DictIdMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
@@ -308,7 +319,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class IntMatcher implements ValueMatcher {
- private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final int[] _buffer = new int[_batchSize];
@Override
public boolean doesValueMatch(int docId) {
@@ -325,7 +336,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class IntMatcherAndNullHandler implements ValueMatcher {
private final ImmutableRoaringBitmap _nullBitmap;
- private final int[] _buffer = new int[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final int[] _buffer = new int[_batchSize];
public IntMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
_nullBitmap = nullBitmap;
@@ -349,7 +360,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class LongMatcher implements ValueMatcher {
- private final long[] _buffer = new long[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final long[] _buffer = new long[_batchSize];
@Override
public boolean doesValueMatch(int docId) {
@@ -366,7 +377,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class LongMatcherAndNullHandler implements ValueMatcher {
private final ImmutableRoaringBitmap _nullBitmap;
- private final long[] _buffer = new long[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final long[] _buffer = new long[_batchSize];
public LongMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
_nullBitmap = nullBitmap;
@@ -390,7 +401,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class FloatMatcher implements ValueMatcher {
- private final float[] _buffer = new float[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final float[] _buffer = new float[_batchSize];
@Override
public boolean doesValueMatch(int docId) {
@@ -407,7 +418,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class FloatMatcherAndNullHandler implements ValueMatcher {
private final ImmutableRoaringBitmap _nullBitmap;
- private final float[] _buffer = new float[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final float[] _buffer = new float[_batchSize];
public FloatMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
_nullBitmap = nullBitmap;
@@ -431,7 +442,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class DoubleMatcher implements ValueMatcher {
- private final double[] _buffer = new double[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final double[] _buffer = new double[_batchSize];
@Override
public boolean doesValueMatch(int docId) {
@@ -448,7 +459,7 @@ public final class SVScanDocIdIterator implements ScanBasedDocIdIterator {
private class DoubleMatcherAndNullHandler implements ValueMatcher {
private final ImmutableRoaringBitmap _nullBitmap;
- private final double[] _buffer = new double[OPTIMAL_ITERATOR_BATCH_SIZE];
+ private final double[] _buffer = new double[_batchSize];
public DoubleMatcherAndNullHandler(ImmutableRoaringBitmap nullBitmap) {
_nullBitmap = nullBitmap;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org