You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ri...@apache.org on 2022/12/29 19:17:34 UTC
[pinot] 01/01: evaluate EQ queries against range index when available
This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a commit to branch range-index-equals-queries
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit d42589a9340f4a889fa89123603c2d3c2e648870
Author: Richard Startin <ri...@apache.org>
AuthorDate: Thu Dec 29 19:16:33 2022 +0000
evaluate EQ queries against range index when available
---
LICENSE-binary | 4 +-
.../core/operator/filter/FilterOperatorUtils.java | 6 +-
.../filter/RangeIndexBasedFilterOperator.java | 194 ++++++++++++++++++++-
.../predicate/EqualsPredicateEvaluatorFactory.java | 99 ++++++++++-
.../org/apache/pinot/queries/RangeQueriesTest.java | 29 +++
.../index/readers/BitSlicedRangeIndexReader.java | 80 +++++++++
.../index/creator/BitSlicedIndexCreatorTest.java | 36 ++++
pom.xml | 2 +-
8 files changed, 434 insertions(+), 16 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index beeb300fd5..4984da9daf 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -453,8 +453,8 @@ org.jetbrains:annotations:13.0
org.lz4:lz4-java:1.8.0
org.objenesis:objenesis:2.1
org.quartz-scheduler:quartz:2.3.2
-org.roaringbitmap:RoaringBitmap:0.9.35
-org.roaringbitmap:shims:0.9.35
+org.roaringbitmap:RoaringBitmap:0.9.37-SNAPSHOT
+org.roaringbitmap:shims:0.9.37-SNAPSHOT
org.scala-lang.modules:scala-collection-compat_2.12:2.3.0
org.scala-lang.modules:scala-java8-compat_2.12:0.9.1
org.scala-lang.modules:scala-xml_2.12:1.3.0
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index c343dcea88..2490ebc50e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -61,7 +61,8 @@ public class FilterOperatorUtils {
if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null) {
return new SortedIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
- if (dataSource.getRangeIndex() != null) {
+ if (dataSource.getRangeIndex() != null
+ && RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
return new RangeIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
return new ScanBasedFilterOperator(predicateEvaluator, dataSource, numDocs, nullHandlingEnabled);
@@ -80,6 +81,9 @@ public class FilterOperatorUtils {
if (dataSource.getInvertedIndex() != null) {
return new BitmapBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
}
+ if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
+ return new RangeIndexBasedFilterOperator(predicateEvaluator, dataSource, numDocs);
+ }
return new ScanBasedFilterOperator(predicateEvaluator, dataSource, numDocs, nullHandlingEnabled);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
index 4b968ba2ed..75ced486a3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/RangeIndexBasedFilterOperator.java
@@ -19,12 +19,15 @@
package org.apache.pinot.core.operator.filter;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
+import java.util.Set;
import org.apache.pinot.core.common.Operator;
import org.apache.pinot.core.operator.blocks.FilterBlock;
import org.apache.pinot.core.operator.dociditerators.ScanBasedDocIdIterator;
import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
import org.apache.pinot.core.operator.docidsets.FilterBlockDocIdSet;
+import org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.DoubleRawValueBasedRangePredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.FloatRawValueBasedRangePredicateEvaluator;
@@ -33,6 +36,7 @@ import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFa
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory.SortedDictionaryBasedRangePredicateEvaluator;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.RangeIndexReader;
+import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.trace.FilterType;
import org.apache.pinot.spi.trace.InvocationRecording;
import org.apache.pinot.spi.trace.Tracing;
@@ -49,6 +53,10 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
private final DataSource _dataSource;
private final int _numDocs;
+ public static boolean canEvaluate(PredicateEvaluator predicateEvaluator, DataSource dataSource) {
+ return RangeEvaluator.canEvaluate(predicateEvaluator, dataSource);
+ }
+
@SuppressWarnings("unchecked")
public RangeIndexBasedFilterOperator(PredicateEvaluator rangePredicateEvaluator, DataSource dataSource, int numDocs) {
_rangePredicateEvaluator = rangePredicateEvaluator;
@@ -131,27 +139,75 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
interface RangeEvaluator {
+
+ Set<FieldSpec.DataType> SUPPORTED_RAW_DATA_TYPES = EnumSet.of(FieldSpec.DataType.INT,
+ FieldSpec.DataType.LONG, FieldSpec.DataType.FLOAT, FieldSpec.DataType.DOUBLE);
+
+ static boolean canEvaluate(PredicateEvaluator predicateEvaluator, DataSource dataSource) {
+ if (dataSource.getRangeIndex() != null) {
+ boolean datatypeSupported = dataSource.getRangeIndex().isExact()
+ && (predicateEvaluator.isDictionaryBased()
+ || SUPPORTED_RAW_DATA_TYPES.contains(predicateEvaluator.getDataType()));
+ switch (predicateEvaluator.getPredicateType()) {
+ case EQ:
+ return datatypeSupported && dataSource.getRangeIndex().isExact()
+ && predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator;
+ case RANGE:
+ return datatypeSupported && (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof IntRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof LongRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof FloatRawValueBasedRangePredicateEvaluator
+ || predicateEvaluator instanceof DoubleRawValueBasedRangePredicateEvaluator);
+ default:
+ }
+ }
+ return false;
+ }
+
static RangeEvaluator of(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
PredicateEvaluator predicateEvaluator) {
- if (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator) {
- return new IntRangeEvaluator(rangeIndexReader,
- (SortedDictionaryBasedRangePredicateEvaluator) predicateEvaluator);
+ if (predicateEvaluator.isDictionaryBased()) {
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new IntPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
+ if (predicateEvaluator instanceof SortedDictionaryBasedRangePredicateEvaluator) {
+ return new IntRangeEvaluator(rangeIndexReader,
+ (SortedDictionaryBasedRangePredicateEvaluator) predicateEvaluator);
+ }
+ throw new IllegalStateException("Unsorted dictionary not supported for Range Indexing");
} else {
switch (predicateEvaluator.getDataType()) {
case INT:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new IntPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new IntRangeEvaluator(rangeIndexReader,
(IntRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case LONG:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new LongPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new LongRangeEvaluator(rangeIndexReader,
(LongRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case FLOAT:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new FloatPointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new FloatRangeEvaluator(rangeIndexReader,
(FloatRawValueBasedRangePredicateEvaluator) predicateEvaluator);
case DOUBLE:
+ if (predicateEvaluator instanceof EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) {
+ return new DoublePointEvaluator(rangeIndexReader,
+ (EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator) predicateEvaluator);
+ }
return new DoubleRangeEvaluator(rangeIndexReader,
(DoubleRawValueBasedRangePredicateEvaluator) predicateEvaluator);
default:
- throw new IllegalStateException("String and Bytes data type not supported for Range Indexing");
+ throw new IllegalStateException("BigDecimal and String, Bytes data type not supported for Range Indexing");
}
}
}
@@ -208,6 +264,40 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class IntPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final int _value;
+
+ private IntPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.isDictionaryBased()
+ ? predicateEvaluator.getMatchingDictId()
+ : predicateEvaluator.getMatchingIntValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class LongRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final long _min;
@@ -241,6 +331,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class LongPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final long _value;
+
+ private LongPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingLongValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class FloatRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final float _min;
@@ -274,6 +396,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class FloatPointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final float _value;
+
+ private FloatPointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingFloatValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private static final class DoubleRangeEvaluator implements RangeEvaluator {
final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
final double _min;
@@ -307,6 +461,38 @@ public class RangeIndexBasedFilterOperator extends BaseFilterOperator {
}
}
+ private static final class DoublePointEvaluator implements RangeEvaluator {
+
+ final RangeIndexReader<ImmutableRoaringBitmap> _rangeIndexReader;
+ final double _value;
+
+ private DoublePointEvaluator(RangeIndexReader<ImmutableRoaringBitmap> rangeIndexReader,
+ EqualsPredicateEvaluatorFactory.EqualsPredicateEvaluator predicateEvaluator) {
+ _rangeIndexReader = rangeIndexReader;
+ _value = predicateEvaluator.getMatchingDoubleValue();
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds() {
+ return _rangeIndexReader.getMatchingDocIds(_value);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getPartiallyMatchingDocIds() {
+ return null;
+ }
+
+ @Override
+ public int getNumMatchingDocs() {
+ return _rangeIndexReader.getNumMatchingDocs(_value);
+ }
+
+ @Override
+ public boolean isExact() {
+ return true;
+ }
+ }
+
private void recordFilter(ImmutableRoaringBitmap bitmap) {
InvocationRecording recording = Tracing.activeRecording();
if (recording.isEnabled()) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
index 814a45253b..7b42c71f9b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
@@ -35,6 +35,41 @@ public class EqualsPredicateEvaluatorFactory {
private EqualsPredicateEvaluatorFactory() {
}
+ public interface EqualsPredicateEvaluator extends PredicateEvaluator {
+
+ default int getMatchingDictId() {
+ throw new UnsupportedOperationException();
+ }
+
+ default int getMatchingIntValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default long getMatchingLongValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default float getMatchingFloatValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default double getMatchingDoubleValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default BigDecimal getMatchingBigDecimalValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default String getMatchingStringValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ default byte[] getMatchingBytesValue() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
/**
* Create a new instance of dictionary based EQ predicate evaluator.
*
@@ -82,7 +117,8 @@ public class EqualsPredicateEvaluatorFactory {
}
}
- private static final class DictionaryBasedEqPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator {
+ private static final class DictionaryBasedEqPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final int _matchingDictId;
final int[] _matchingDictIds;
@@ -128,9 +164,15 @@ public class EqualsPredicateEvaluatorFactory {
public int[] getMatchingDictIds() {
return _matchingDictIds;
}
+
+ @Override
+ public int getMatchingDictId() {
+ return _matchingDictId;
+ }
}
- private static final class IntRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class IntRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final int _matchingValue;
IntRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, int matchingValue) {
@@ -165,9 +207,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public int getMatchingIntValue() {
+ return _matchingValue;
+ }
}
- private static final class LongRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class LongRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final long _matchingValue;
LongRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, long matchingValue) {
@@ -202,9 +250,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public long getMatchingLongValue() {
+ return _matchingValue;
+ }
}
- private static final class FloatRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class FloatRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final float _matchingValue;
FloatRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, float matchingValue) {
@@ -239,9 +293,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public float getMatchingFloatValue() {
+ return _matchingValue;
+ }
}
- private static final class DoubleRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class DoubleRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final double _matchingValue;
DoubleRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, double matchingValue) {
@@ -276,9 +336,15 @@ public class EqualsPredicateEvaluatorFactory {
}
return matches;
}
+
+ @Override
+ public double getMatchingDoubleValue() {
+ return _matchingValue;
+ }
}
- private static final class BigDecimalRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class BigDecimalRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final BigDecimal _matchingValue;
BigDecimalRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, BigDecimal matchingValue) {
@@ -300,9 +366,15 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(BigDecimal value) {
return _matchingValue.compareTo(value) == 0;
}
+
+ @Override
+ public BigDecimal getMatchingBigDecimalValue() {
+ return _matchingValue;
+ }
}
- private static final class StringRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class StringRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final String _matchingValue;
StringRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, String matchingValue) {
@@ -324,9 +396,15 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(String value) {
return _matchingValue.equals(value);
}
+
+ @Override
+ public String getMatchingStringValue() {
+ return _matchingValue;
+ }
}
- private static final class BytesRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ private static final class BytesRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator
+ implements EqualsPredicateEvaluator {
final byte[] _matchingValue;
BytesRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, byte[] matchingValue) {
@@ -348,5 +426,10 @@ public class EqualsPredicateEvaluatorFactory {
public boolean applySV(byte[] value) {
return Arrays.equals(_matchingValue, value);
}
+
+ @Override
+ public byte[] getMatchingBytesValue() {
+ return _matchingValue;
+ }
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
index 44ba46b2db..099981d681 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
@@ -145,6 +145,16 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildSelectionQuery(RAW_LONG_COL, 250, 500, false), 250, 500, false},
{buildSelectionQuery(RAW_FLOAT_COL, 250, 500, false), 250, 500, false},
{buildSelectionQuery(RAW_DOUBLE_COL, 250, 500, false), 250, 500, false},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_INT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_LONG_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 300), 300, 300, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 300), 300, 300, true},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_INT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_LONG_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 301), 301, 301, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 301), 301, 301, true}
};
}
@@ -158,6 +168,11 @@ public class RangeQueriesTest extends BaseQueriesTest {
}
}
+ private static String buildSelectionQuery(String filterCol, Number value) {
+ return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " = "
+ + formatValue(filterCol, value);
+ }
+
@DataProvider
public static Object[][] countTestCases() {
return new Object[][]{
@@ -171,6 +186,16 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildCountQuery(RAW_LONG_COL, 250, 500, false), 2},
{buildCountQuery(RAW_FLOAT_COL, 250, 500, false), 2},
{buildCountQuery(RAW_DOUBLE_COL, 250, 500, false), 2},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 300), 1},
+ {buildCountQuery(RAW_INT_COL, 300), 1},
+ {buildCountQuery(RAW_LONG_COL, 300), 1},
+ {buildCountQuery(RAW_FLOAT_COL, 300), 1},
+ {buildCountQuery(RAW_DOUBLE_COL, 300), 1},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 301), 0},
+ {buildCountQuery(RAW_INT_COL, 301), 0},
+ {buildCountQuery(RAW_LONG_COL, 301), 0},
+ {buildCountQuery(RAW_FLOAT_COL, 301), 0},
+ {buildCountQuery(RAW_DOUBLE_COL, 301), 0}
};
}
@@ -184,6 +209,10 @@ public class RangeQueriesTest extends BaseQueriesTest {
}
}
+ private static String buildCountQuery(String filterCol, Number value) {
+ return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " = " + formatValue(filterCol, value);
+ }
+
private static String buildFilter(String filterCol, Number min, Number max) {
switch (filterCol) {
case DICTIONARIZED_INT_COL:
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
index 0fde2d9ddc..2aa302a924 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
@@ -94,6 +94,32 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
return queryRangeBitmapCardinality(FPOrdering.ordinalOf(min), FPOrdering.ordinalOf(max), 0xFFFFFFFFFFFFFFFFL);
}
+ @Override
+ public int getNumMatchingDocs(int value) {
+ if (value < _min) {
+ return 0;
+ }
+ return queryRangeBitmapCardinality(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Override
+ public int getNumMatchingDocs(long value) {
+ if (value < _min) {
+ return 0;
+ }
+ return queryRangeBitmapCardinality(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Override
+ public int getNumMatchingDocs(float value) {
+ return queryRangeBitmapCardinality(FPOrdering.ordinalOf(value), 0xFFFFFFFFL);
+ }
+
+ @Override
+ public int getNumMatchingDocs(double value) {
+ return queryRangeBitmapCardinality(FPOrdering.ordinalOf(value), 0xFFFFFFFFFFFFFFFFL);
+ }
+
@Override
public ImmutableRoaringBitmap getMatchingDocIds(int min, int max) {
// TODO: Handle this before reading the range index
@@ -130,6 +156,36 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
return queryRangeBitmap(FPOrdering.ordinalOf(min), FPOrdering.ordinalOf(max), 0xFFFFFFFFFFFFFFFFL);
}
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(int value) {
+ if (value < _min) {
+ return new MutableRoaringBitmap();
+ }
+ return queryRangeBitmap(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(long value) {
+ if (value < _min) {
+ return new MutableRoaringBitmap();
+ }
+ return queryRangeBitmap(Math.max(value, _min) - _min, _max - _min);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(float value) {
+ return queryRangeBitmap(FPOrdering.ordinalOf(value), 0xFFFFFFFFL);
+ }
+
+ @Nullable
+ @Override
+ public ImmutableRoaringBitmap getMatchingDocIds(double value) {
+ return queryRangeBitmap(FPOrdering.ordinalOf(value), 0xFFFFFFFFFFFFFFFFL);
+ }
+
// this index supports exact matches, so always return null for partial matches
@Nullable
@@ -160,6 +216,9 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
RangeBitmap rangeBitmap = mapRangeBitmap();
if (Long.compareUnsigned(max, columnMax) < 0) {
if (Long.compareUnsigned(min, 0) > 0) {
+ if (min == max) {
+ return rangeBitmap.eq(min).toMutableRoaringBitmap();
+ }
return rangeBitmap.between(min, max).toMutableRoaringBitmap();
}
return rangeBitmap.lte(max).toMutableRoaringBitmap();
@@ -173,10 +232,22 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
}
}
+ private ImmutableRoaringBitmap queryRangeBitmap(long value, long columnMax) {
+ RangeBitmap rangeBitmap = mapRangeBitmap();
+ if (Long.compareUnsigned(value, columnMax) < 0) {
+ return rangeBitmap.eq(value).toMutableRoaringBitmap();
+ } else {
+ return new MutableRoaringBitmap();
+ }
+ }
+
private int queryRangeBitmapCardinality(long min, long max, long columnMax) {
RangeBitmap rangeBitmap = mapRangeBitmap();
if (Long.compareUnsigned(max, columnMax) < 0) {
if (Long.compareUnsigned(min, 0) > 0) {
+ if (min == max) {
+ return (int) rangeBitmap.eqCardinality(min);
+ }
return (int) rangeBitmap.betweenCardinality(min, max);
}
return (int) rangeBitmap.lteCardinality(max);
@@ -188,6 +259,15 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
}
}
+ private int queryRangeBitmapCardinality(long value, long columnMax) {
+ RangeBitmap rangeBitmap = mapRangeBitmap();
+ if (Long.compareUnsigned(value, columnMax) < 0) {
+ return (int) rangeBitmap.eqCardinality(value);
+ } else {
+ return 0;
+ }
+ }
+
private RangeBitmap mapRangeBitmap() {
// note that this is a very cheap operation, no deserialization is required
ByteBuffer buffer = _dataBuffer.toDirectByteBuffer(_offset, (int) (_dataBuffer.size() - _offset));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
index 28513e1766..d80723c352 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/BitSlicedIndexCreatorTest.java
@@ -171,9 +171,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Integer.MAX_VALUE);
testRange(reader, dataset, Integer.MAX_VALUE, Integer.MAX_VALUE);
testRange(reader, dataset, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ testPoint(reader, dataset, Integer.MIN_VALUE);
+ testPoint(reader, dataset, Integer.MAX_VALUE);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -192,6 +195,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<int[]> dataset, int value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testLong(Dataset<long[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -216,9 +225,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Long.MAX_VALUE);
testRange(reader, dataset, Long.MAX_VALUE, Long.MAX_VALUE);
testRange(reader, dataset, Long.MIN_VALUE, Long.MAX_VALUE);
+ testPoint(reader, dataset, Long.MIN_VALUE);
+ testPoint(reader, dataset, Long.MAX_VALUE);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -237,6 +249,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<long[]> dataset, long value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testFloat(Dataset<float[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -261,9 +279,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Float.POSITIVE_INFINITY);
testRange(reader, dataset, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY);
testRange(reader, dataset, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Float.NEGATIVE_INFINITY);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -282,6 +303,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<float[]> dataset, float value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private void testDouble(Dataset<double[]> dataset)
throws IOException {
ColumnMetadata metadata = dataset.toColumnMetadata();
@@ -306,9 +333,12 @@ public class BitSlicedIndexCreatorTest {
}
testRange(reader, dataset, prev, prev + 1);
testRange(reader, dataset, prev + 1, prev + 1);
+ testPoint(reader, dataset, prev + 1);
testRange(reader, dataset, prev + 1, Double.POSITIVE_INFINITY);
testRange(reader, dataset, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY);
testRange(reader, dataset, Double.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Double.POSITIVE_INFINITY);
+ testPoint(reader, dataset, Double.NEGATIVE_INFINITY);
} finally {
FileUtils.forceDelete(rangeIndexFile);
}
@@ -327,6 +357,12 @@ public class BitSlicedIndexCreatorTest {
}
}
+ private static void testPoint(BitSlicedRangeIndexReader reader, Dataset<double[]> dataset, double value) {
+ ImmutableRoaringBitmap reference = dataset.scan(value, value);
+ assertEquals(reader.getMatchingDocIds(value), reference);
+ assertEquals(reader.getNumMatchingDocs(value), reference.getCardinality());
+ }
+
private static BitSlicedRangeIndexCreator newBitSlicedIndexCreator(ColumnMetadata metadata) {
return metadata.hasDictionary() ? new BitSlicedRangeIndexCreator(INDEX_DIR,
metadata.getFieldSpec(), metadata.getCardinality()) : new BitSlicedRangeIndexCreator(INDEX_DIR,
diff --git a/pom.xml b/pom.xml
index 5b0bd30e3e..975044d2b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -442,7 +442,7 @@
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
- <version>0.9.35</version>
+ <version>0.9.37-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.101tec</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org