You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ri...@apache.org on 2022/12/29 19:17:33 UTC
[pinot] branch range-index-equals-queries created (now d42589a934)
This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a change to branch range-index-equals-queries
in repository https://gitbox.apache.org/repos/asf/pinot.git
at d42589a934 evaluate EQ queries against range index when available
This branch includes the following new commits:
new d42589a934 evaluate EQ queries against range index when available
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[pinot] 01/01: evaluate EQ queries against range index when available
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a commit to branch range-index-equals-queries
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit d42589a9340f4a889fa89123603c2d3c2e648870
Author: Richard Startin <ri...@apache.org>
AuthorDate: Thu Dec 29 19:16:33 2022 +0000
evaluate EQ queries against range index when available
---
LICENSE-binary | 4 +-
.../core/operator/filter/FilterOperatorUtils.java | 6 +-
.../filter/RangeIndexBasedFilterOperator.java | 194 ++++++++++++++++++++-
.../predicate/EqualsPredicateEvaluatorFactory.java | 99 ++++++++++-
.../org/apache/pinot/queries/RangeQueriesTest.java | 29 +++
.../index/readers/BitSlicedRangeIndexReader.java | 80 +++++++++
.../index/creator/BitSlicedIndexCreatorTest.java | 36 ++++
pom.xml | 2 +-
8 files changed, 434 insertions(+), 16 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index beeb300fd5..4984da9daf 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -453,8 +453,8 @@ org.jetbrains:annotations:13.0
org.lz4:lz4-java:1.8.0
org.objenesis:objenesis:2.1
org.quartz-scheduler:quartz:2.3.2
-org.roaringbitmap:RoaringBitmap:0.9.35
-org.roaringbitmap:shims:0.9.35
+org.roaringbitmap:RoaringBitmap:0.9.37-SNAPSHOT
+org.roaringbitmap:shims:0.9.37-SNAPSHOT
org.scala-lang.modules:scala-collection-compat_2.12:2.3.0
org.scala-lang.modules:scala-java8-compat_2.12:0.9.1
org.scala-lang.modules:scala-xml_2.12:1.3.0
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index c343dcea88..2490ebc50e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -61,7 +61,8 @@ public class FilterOperatorUtils {
if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null) {
return new SortedIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
- if (dataSource.getRangeIndex() != null) {
+ if (dataSource.getRangeIndex() != null
+ && RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
return new RangeIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
return new ScanBasedFilterOperator(predicateEvaluator, dataSource, numDocs, nullHandlingEnabled);
@@ -80,6 +81,9 @@ public class FilterOperatorUtils {
if (dataSource.getInvertedIndex() != null) {
return new BitmapBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
+ if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
+ return new RangeIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
+ }
return new ScanBasedFilterOperator(predicateEvaluator, dataSource, numDocs, nullHandlingEnabled);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
index 4b968ba2ed..75ced486a3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
@@ -19,12 +19,15 @@
package org.apache.pinot.core.operator.filter;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
+import java.util.Set;
import org.apache.pinot.core.common.Operator;
import org.apache.pinot.core.operator.blocks.FilterBlock;
import org.apache.pinot.core.operator.dociditerators.ScanBasedDocIdIterator;
import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
import org.apache.pinot.core.operator.docidsets.FilterBlockDocIdSet;
+import org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.DoubleRawValueBasedRangePredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.FloatRawValueBasedRangePredicateEvaluator;
@@ -33,6 +36,7 @@ import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFa
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.SortedDictionaryBasedRangePredicateEvaluator;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.RangeIndexReader;
+import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.trace.FilterType;
import org.apache.pinot.spi.trace.InvocationRecording;
import org.apache.pinot.spi.trace.Tracing;
@@ -49,6 +53,10 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
private final DataSource _dataSource;
private final int _numDocs;
+ public static boolean canEvaluate(PredicateEvaluator predicateEvaluator, DataSource dataSource) {
+ return RangeEvaluator.canEvaluate(predicateEvaluator, dataSource);
+ }
+
@SuppressWarnings("unchecked")
public RangeIndexBasedFilterOperator(PredicateEvaluator rangePredicateEvaluator, DataSource dataSource, int numDocs) {
_rangePredicateEvaluator = rangePredicateEvaluator;
@@ -131,27 +139,75 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
interface RangeEvaluator {
+
+ Set<FieldSpec.DataType> SUPPORTED_RAW_DATA_TYPES = EnumSet.of(FieldSpec.DataType.INT,
+ FieldSpec.DataType.LONG, FieldSpec.DataType.FLOAT, FieldSpec.DataType.DOUBLE);
+
+ static boolean canEvaluate(PredicateEvaluator predicateEvaluator, DataSource dataSource) {
+ if (dataSource.getRangeIndex() != null) {
+ boolean datatypeSupported = dataSource.getRangeIndex().isExact()
+ && (predicateEvaluator.isDictionaryBased()
+ || SUPPORTED_RAW_DATA_TYPES.contains(predicateEvaluator.getDataType()));
+ switch (predicateEvaluator.getPredicateType()) {
+ case EQ:
+ return datatypeSupported && dataSource.getRangeIndex().isExact()
+ && predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator;
+ case RANGE:
+ return datatypeSupported && (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof IntRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof LongRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof FloatRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof DoubleRawValueBasedRangePredicateEvaluator);
+ default:
+ }
+ }
+ return false;
+ }
+
static RangeEvaluator of(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
PredicateEvaluator predicateEvaluator) {
- if (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator) {
- return new IntRangeEvaluator(rangeIndexReader,
- (SortedDictionaryBasedRangePredicateEvaluator) predicateEvaluator);
+ if (predicateEvaluator.isDictionaryBased()) {
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new IntPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
+ if (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator) {
+ return new IntRangeEvaluator(rangeIndexReader,
+ (SortedDictionaryBasedRangePredicateEvaluator) predicateEvaluator);
+ }
+ throw new IllegalStateException("Unsorted dictionary not supported for Range Indexing");
} else {
switch (predicateEvaluator.getDataType()) {
case INT:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new IntPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new IntRangeEvaluator(rangeIndexReader,
(IntRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case LONG:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new LongPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new LongRangeEvaluator(rangeIndexReader,
(LongRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case FLOAT:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new FloatPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new FloatRangeEvaluator(rangeIndexReader,
(FloatRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case DOUBLE:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new DoublePointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new DoubleRangeEvaluator(rangeIndexReader,
(DoubleRawValueBasedRangePredicateEvaluator) predicateEvaluator);
default:
- throw new IllegalStateException("String and Bytes data type not supported for Range Indexing");
+ throw new IllegalStateException("BigDecimal and String, Bytes data type not supported for Range Indexing");
}
}
}
@@ -208,6 +264,40 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class IntPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final int _value;
+
+ private IntPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.isDictionaryBased()
+ ? predicateEvaluator.getMatchingDictId()
+ : predicateEvaluator.getMatchingIntValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class LongRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final long _min;
@@ -241,6 +331,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class LongPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final long _value;
+
+ private LongPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingLongValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class FloatRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final float _min;
@@ -274,6 +396,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class FloatPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final float _value;
+
+ private FloatPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingFloatValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class DoubleRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final double _min;
@@ -307,6 +461,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class DoublePointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final double _value;
+
+ private DoublePointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingDoubleValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private void recordFilter(ImmutableRoaringBitmap bitmap) {
InvocationRecording recording = Tracing.activeRecording();
if (recording.isEnabled()) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
index 814a45253b..7b42c71f9b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
@@ -35,6 +35,41 @@ public class EqualsPredicateEvaluatorFactory {
private EqualsPredicateEvaluatorFactory() {
}
+ public interface EqualsPredicateEvaluator extends PredicateEvaluator {
+
+ default int getMatchingDictId() {
+ throw new UnsupportedOperationException();
+ }
+
+ default int getMatchingIntValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default long getMatchingLongValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default float getMatchingFloatValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default double getMatchingDoubleValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default BigDecimal getMatchingBigDecimalValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default String getMatchingStringValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default byte[] getMatchingBytesValue() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
/**
* Create a new instance of dictionary based EQ predicate evaluator.
*
@@ -82,7 +117,8 @@ public class EqualsPredicateEvaluatorFactory {
}
}
- private static final class DictionaryBasedEqPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator {
+ private static final class DictionaryBasedEqPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final int _matchingDictId;
final int[] _matchingDictIds;
@@ -128,9 +164,15 @@ public class EqualsPredicateEvaluatorFactory {
public int[] getMatchingDictIds() {
return _matchingDictIds;
}
+
+ @Override
+ public int getMatchingDictId() {
+ return _matchingDictId;
+ }
}
- private static final class IntRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class IntRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final int _matchingValue;
IntRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, int matchingValue) {
@@ -165,9 +207,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public int getMatchingIntValue() {
+ return _matchingValue;
+ }
}
- private static final class LongRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class LongRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final long _matchingValue;
LongRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, long matchingValue) {
@@ -202,9 +250,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public long getMatchingLongValue() {
+ return _matchingValue;
+ }
}
- private static final class FloatRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class FloatRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final float _matchingValue;
FloatRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, float matchingValue) {
@@ -239,9 +293,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public float getMatchingFloatValue() {
+ return _matchingValue;
+ }
}
- private static final class DoubleRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class DoubleRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final double _matchingValue;
DoubleRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, double matchingValue) {
@@ -276,9 +336,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public double getMatchingDoubleValue() {
+ return _matchingValue;
+ }
}
- private static final class BigDecimalRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class BigDecimalRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final BigDecimal _matchingValue;
BigDecimalRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, BigDecimal matchingValue) {
@@ -300,9 +366,15 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(BigDecimal value) {
return _matchingValue.compareTo(value) == 0;
}
+
+ @Override
+ public BigDecimal getMatchingBigDecimalValue() {
+ return _matchingValue;
+ }
}
- private static final class StringRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class StringRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final String _matchingValue;
StringRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, String matchingValue) {
@@ -324,9 +396,15 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(String value) {
return _matchingValue.equals(value);
}
+
+ @Override
+ public String getMatchingStringValue() {
+ return _matchingValue;
+ }
}
- private static final class BytesRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class BytesRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final byte[] _matchingValue;
BytesRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, byte[] matchingValue) {
@@ -348,5 +426,10 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(byte[] value) {
return Arrays.equals(_matchingValue, value);
}
+
+ @Override
+ public byte[] getMatchingBytesValue() {
+ return _matchingValue;
+ }
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
index 44ba46b2db..099981d681 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
@@ -145,6 +145,16 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildSelectionQuery(RAW_LONG_COL, 250, 500, false), 250, 500, false},
{buildSelectionQuery(RAW_FLOAT_COL, 250, 500, false), 250, 500, false},
{buildSelectionQuery(RAW_DOUBLE_COL, 250, 500, false), 250, 500, false},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_INT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_LONG_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 300), 300, 300, true},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_INT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_LONG_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 301), 301, 301, true}
};
}
@@ -158,6 +168,11 @@ public class RangeQueriesTest extends BaseQueriesTest {
}
}
+ private static String buildSelectionQuery(String filterCol, Number value) {
+ return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " = "
+ + formatValue(filterCol, value);
+ }
+
@DataProvider
public static Object[][] countTestCases() {
return new Object[][]{
@@ -171,6 +186,16 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildCountQuery(RAW_LONG_COL, 250, 500, false), 2},
{buildCountQuery(RAW_FLOAT_COL, 250, 500, false), 2},
{buildCountQuery(RAW_DOUBLE_COL, 250, 500, false), 2},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 300), 1},
+ {buildCountQuery(RAW_INT_COL, 300), 1},
+ {buildCountQuery(RAW_LONG_COL, 300), 1},
+ {buildCountQuery(RAW_FLOAT_COL, 300), 1},
+ {buildCountQuery(RAW_DOUBLE_COL, 300), 1},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 301), 0},
+ {buildCountQuery(RAW_INT_COL, 301), 0},
+ {buildCountQuery(RAW_LONG_COL, 301), 0},
+ {buildCountQuery(RAW_FLOAT_COL, 301), 0},
+ {buildCountQuery(RAW_DOUBLE_COL, 301), 0}
};
}
@@ -184,6 +209,10 @@ public class RangeQueriesTest extends BaseQueriesTest {
}
}
+ private static String buildCountQuery(String filterCol, Number value) {
+ return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " = " + formatValue(filterCol, value);
+ }
+
private static String buildFilter(String filterCol, Number min, Number max) {
switch (filterCol) {
case DICTIONARIZED_INT_COL:
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
index 0fde2d9ddc..2aa302a924 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
@@ -94,6 +94,32 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
return queryRangeBitmapCardinality(FPOrdering.ordinalOf(min), FPOrdering.ordinalOf(max), 0xFFFFFFFFFFFFFFFFL);
}
+ @Override
+ public int getNumMatchingDocs(int value) {
+ if (value < _min) {
+ return 0;
+ }
+ return queryRangeBitmapCardinality(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Override
+ public int getNumMatchingDocs(long value) {
+ if (value < _min) {
+ return 0;
+ }
+ return queryRangeBitmapCardinality(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Override
+ public int getNumMatchingDocs(float value) {
+ return queryRangeBitmapCardinality(FPOrdering.ordinalOf(value), 0xFFFFFFFFL);
+ }
+
+ @Override
+ public int getNumMatchingDocs(double value) {
+ return queryRangeBitmapCardinality(FPOrdering.ordinalOf(value), 0xFFFFFFFFFFFFFFFFL);
+ }
+
@Override
public ImmutableRoaringBitmap getMatchingDocIds(int min, int max) {
// TODO: Handle this before reading the range index
@@ -130,6 +156,36 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
return queryRangeBitmap(FPOrdering.ordinalOf(min), FPOrdering.ordinalOf(max), 0xFFFFFFFFFFFFFFFFL);
}
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(int value) {
+ if (value < _min) {
+ return new MutableRoaringBitmap();
+ }
+ return queryRangeBitmap(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(long value) {
+ if (value < _min) {
+ return new MutableRoaringBitmap();
+ }
+ return queryRangeBitmap(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(float value) {
+ return queryRangeBitmap(FPOrdering.ordinalOf(value), 0xFFFFFFFFL);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(double value) {
+ return queryRangeBitmap(FPOrdering.ordinalOf(value), 0xFFFFFFFFFFFFFFFFL);
+ }
+
// this index supports exact matches, so always return null for partial matches
@Nullable
@@ -160,6 +216,9 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
RangeBitmap rangeBitmap = mapRangeBitmap();
if (Long.compareUnsigned(max, columnMax) < 0) {
if (Long.compareUnsigned(min, 0) > 0) {
+ if (min == max) {
+ return rangeBitmap.eq(min).toMutableRoaringBitmap();
+ }
return rangeBitmap.between(min, max).toMutableRoaringBitmap();
}
return rangeBitmap.lte(max).toMutableRoaringBitmap();
@@ -173,10 +232,22 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
}
}
+ private ImmutableRoaringBitmap queryRangeBitmap(long value, long columnMax) {
+ RangeBitmap rangeBitmap = mapRangeBitmap();
+ if (Long.compareUnsigned(value, columnMax) < 0) {
+ return rangeBitmap.eq(value).toMutableRoaringBitmap();
+ } else {
+ return new MutableRoaringBitmap();
+ }
+ }
+
private int queryRangeBitmapCardinality(long min, long max, long columnMax) {
RangeBitmap rangeBitmap = mapRangeBitmap();
if (Long.compareUnsigned(max, columnMax) < 0) {
if (Long.compareUnsigned(min, 0) > 0) {
+ if (min == max) {
+ return (int) rangeBitmap.eqCardinality(min);
+ }
return (int) rangeBitmap.betweenCardinality(min, max);
}
return (int) rangeBitmap.lteCardinality(max);
@@ -188,6 +259,15 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
}
}
+ private int queryRangeBitmapCardinality(long value, long columnMax) {
+ RangeBitmap rangeBitmap = mapRangeBitmap();
+ if (Long.compareUnsigned(value, columnMax) < 0) {
+ return (int) rangeBitmap.eqCardinality(value);
+ } else {
+ return 0;
+ }
+ }
+
private RangeBitmap mapRangeBitmap() {
// note that this is a very cheap operation, no deserialization is required
ByteBuffer buffer = _dataBuffer.toDirectByteBuffer(_offset, (int) (_dataBuffer.size() - _offset));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
index 28513e1766..d80723c352 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
@@ -171,9 +171,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Integer.MAX_VALUE);
testRange(reader, dataset, Integer.MAX_VALUE, Integer.MAX_VALUE);
testRange(reader, dataset, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ testPoint(reader, dataset, Integer.MIN_VALUE);
+ testPoint(reader, dataset, Integer.MAX_VALUE);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -192,6 +195,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<int[]> dataset, int value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testLong(Dataset<long[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -216,9 +225,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Long.MAX_VALUE);
testRange(reader, dataset, Long.MAX_VALUE, Long.MAX_VALUE);
testRange(reader, dataset, Long.MIN_VALUE, Long.MAX_VALUE);
+ testPoint(reader, dataset, Long.MIN_VALUE);
+ testPoint(reader, dataset, Long.MAX_VALUE);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -237,6 +249,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<long[]> dataset, long value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testFloat(Dataset<float[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -261,9 +279,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Float.POSITIVE_INFINITY);
testRange(reader, dataset, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY);
testRange(reader, dataset, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Float.NEGATIVE_INFINITY);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -282,6 +303,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<float[]> dataset, float value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testDouble(Dataset<double[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -306,9 +333,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Double.POSITIVE_INFINITY);
testRange(reader, dataset, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY);
testRange(reader, dataset, Double.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Double.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Double.NEGATIVE_INFINITY);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -327,6 +357,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<double[]> dataset, double value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private static BitSlicedRangeIndexCreator newBitSlicedIndexCreator(ColumnMetadata metadata) {
return metadata.hasDictionary() ? new BitSlicedRangeIndexCreator(INDEX_DIR,
metadata.getFieldSpec(), metadata.getCardinality()) : new BitSlicedRangeIndexCreator(INDEX_DIR,
diff --git a/pom.xml b/pom.xml
index 5b0bd30e3e..975044d2b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -442,7 +442,7 @@
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
- <version>0.9.35</version>
+ <version>0.9.37-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.101tec</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org