You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/07/21 08:39:30 UTC
[pinot] branch master updated: Fix a bug when use range index to solve EQ predicate (#11146)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 12ddd3310b Fix a bug when use range index to solve EQ predicate (#11146)
12ddd3310b is described below
commit 12ddd3310b65c2e3e871a8fba0c8ccc503afd980
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Fri Jul 21 01:39:22 2023 -0700
Fix a bug when use range index to solve EQ predicate (#11146)
---
.../org/apache/pinot/queries/RangeQueriesTest.java | 99 ++++++++++++++++------
.../index/readers/BitSlicedRangeIndexReader.java | 4 +-
2 files changed, 73 insertions(+), 30 deletions(-)
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
index 8dca7d7786..86ac43379b 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
@@ -51,7 +51,6 @@ import static org.testng.Assert.assertTrue;
public class RangeQueriesTest extends BaseQueriesTest {
-
private static final File INDEX_DIR = new File(FileUtils.getTempDirectory(), "RangeQueriesTest");
private static final String RAW_TABLE_NAME = "testTable";
private static final String SEGMENT_NAME = "testSegment";
@@ -64,20 +63,17 @@ public class RangeQueriesTest extends BaseQueriesTest {
private static final String RAW_FLOAT_COL = "rawFloatCol";
private static final String RAW_DOUBLE_COL = "rawDoubleCol";
- private static final Schema SCHEMA = new Schema.SchemaBuilder()
- .addSingleValueDimension(DICTIONARIZED_INT_COL, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_INT_COL, FieldSpec.DataType.INT)
- .addSingleValueDimension(RAW_LONG_COL, FieldSpec.DataType.LONG)
- .addSingleValueDimension(RAW_FLOAT_COL, FieldSpec.DataType.FLOAT)
- .addSingleValueDimension(RAW_DOUBLE_COL, FieldSpec.DataType.DOUBLE)
- .build();
-
- private static final TableConfig TABLE_CONFIG =
- new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
- .setNoDictionaryColumns(Arrays.asList(RAW_INT_COL, RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL))
- .setRangeIndexColumns(Arrays.asList(DICTIONARIZED_INT_COL, RAW_INT_COL, RAW_LONG_COL, RAW_FLOAT_COL,
- RAW_DOUBLE_COL))
- .build();
+ private static final Schema SCHEMA =
+ new Schema.SchemaBuilder().addSingleValueDimension(DICTIONARIZED_INT_COL, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_INT_COL, FieldSpec.DataType.INT)
+ .addSingleValueDimension(RAW_LONG_COL, FieldSpec.DataType.LONG)
+ .addSingleValueDimension(RAW_FLOAT_COL, FieldSpec.DataType.FLOAT)
+ .addSingleValueDimension(RAW_DOUBLE_COL, FieldSpec.DataType.DOUBLE).build();
+
+ private static final TableConfig TABLE_CONFIG = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(Arrays.asList(RAW_INT_COL, RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL))
+ .setRangeIndexColumns(
+ Arrays.asList(DICTIONARIZED_INT_COL, RAW_INT_COL, RAW_LONG_COL, RAW_FLOAT_COL, RAW_DOUBLE_COL)).build();
private IndexSegment _indexSegment;
private List<IndexSegment> _indexSegments;
@@ -134,6 +130,7 @@ public class RangeQueriesTest extends BaseQueriesTest {
@DataProvider
public static Object[][] selectionTestCases() {
+ //@formatter:off
return new Object[][]{
{buildSelectionQuery(DICTIONARIZED_INT_COL, 250, 500, true), 250, 500, true},
{buildSelectionQuery(RAW_INT_COL, 250, 500, true), 250, 500, true},
@@ -154,27 +151,51 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildSelectionQuery(RAW_INT_COL, 301), 301, 301, true},
{buildSelectionQuery(RAW_LONG_COL, 301), 301, 301, true},
{buildSelectionQuery(RAW_FLOAT_COL, 301), 301, 301, true},
- {buildSelectionQuery(RAW_DOUBLE_COL, 301), 301, 301, true}
+ {buildSelectionQuery(RAW_DOUBLE_COL, 301), 301, 301, true},
+
+ // Boundary value
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 0, 500, true), 0, 500, true},
+ {buildSelectionQuery(RAW_INT_COL, 0, 500, true), 0, 500, true},
+ {buildSelectionQuery(RAW_LONG_COL, 0, 500, true), 0, 500, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 0, 500, true), 0, 500, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 0, 500, true), 0, 500, true},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 99500, 99900, false), 99500, 99900, false},
+ {buildSelectionQuery(RAW_INT_COL, 99500, 99900, false), 99500, 99900, false},
+ {buildSelectionQuery(RAW_LONG_COL, 99500, 99900, false), 99500, 99900, false},
+ {buildSelectionQuery(RAW_FLOAT_COL, 99500, 99900, false), 99500, 99900, false},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 99500, 99900, false), 99500, 99900, false},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 0), 0, 0, true},
+ {buildSelectionQuery(RAW_INT_COL, 0), 0, 0, true},
+ {buildSelectionQuery(RAW_LONG_COL, 0), 0, 0, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 0), 0, 0, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 0), 0, 0, true},
+ {buildSelectionQuery(DICTIONARIZED_INT_COL, 99900), 99900, 99900, true},
+ {buildSelectionQuery(RAW_INT_COL, 99900), 99900, 99900, true},
+ {buildSelectionQuery(RAW_LONG_COL, 99900), 99900, 99900, true},
+ {buildSelectionQuery(RAW_FLOAT_COL, 99900), 99900, 99900, true},
+ {buildSelectionQuery(RAW_DOUBLE_COL, 99900), 99900, 99900, true}
};
+ //@formatter:on
}
private static String buildSelectionQuery(String filterCol, Number min, Number max, boolean inclusive) {
if (inclusive) {
- return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " between "
- + buildFilter(filterCol, min, max);
+ return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " between " + buildFilter(
+ filterCol, min, max);
} else {
- return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " > "
- + formatValue(filterCol, min) + " and " + filterCol + " < " + formatValue(filterCol, max);
+ return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " > " + formatValue(
+ filterCol, min) + " and " + filterCol + " < " + formatValue(filterCol, max);
}
}
private static String buildSelectionQuery(String filterCol, Number value) {
- return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " = "
- + formatValue(filterCol, value);
+ return "select " + RAW_INT_COL + " from " + RAW_TABLE_NAME + " where " + filterCol + " = " + formatValue(filterCol,
+ value);
}
@DataProvider
public static Object[][] countTestCases() {
+ //@formatter:off
return new Object[][]{
{buildCountQuery(DICTIONARIZED_INT_COL, 250, 500, true), 3},
{buildCountQuery(RAW_INT_COL, 250, 500, true), 3},
@@ -195,17 +216,40 @@ public class RangeQueriesTest extends BaseQueriesTest {
{buildCountQuery(RAW_INT_COL, 301), 0},
{buildCountQuery(RAW_LONG_COL, 301), 0},
{buildCountQuery(RAW_FLOAT_COL, 301), 0},
- {buildCountQuery(RAW_DOUBLE_COL, 301), 0}
+ {buildCountQuery(RAW_DOUBLE_COL, 301), 0},
+
+ // Boundary value
+ {buildCountQuery(DICTIONARIZED_INT_COL, 0, 500, true), 6},
+ {buildCountQuery(RAW_INT_COL, 0, 500, true), 6},
+ {buildCountQuery(RAW_LONG_COL, 0, 500, true), 6},
+ {buildCountQuery(RAW_FLOAT_COL, 0, 500, true), 6},
+ {buildCountQuery(RAW_DOUBLE_COL, 0, 500, true), 6},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 99500, 99900, false), 3},
+ {buildCountQuery(RAW_INT_COL, 99500, 99900, false), 3},
+ {buildCountQuery(RAW_LONG_COL, 99500, 99900, false), 3},
+ {buildCountQuery(RAW_FLOAT_COL, 99500, 99900, false), 3},
+ {buildCountQuery(RAW_DOUBLE_COL, 99500, 99900, false), 3},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 0), 1},
+ {buildCountQuery(RAW_INT_COL, 0), 1},
+ {buildCountQuery(RAW_LONG_COL, 0), 1},
+ {buildCountQuery(RAW_FLOAT_COL, 0), 1},
+ {buildCountQuery(RAW_DOUBLE_COL, 0), 1},
+ {buildCountQuery(DICTIONARIZED_INT_COL, 99900), 1},
+ {buildCountQuery(RAW_INT_COL, 99900), 1},
+ {buildCountQuery(RAW_LONG_COL, 99900), 1},
+ {buildCountQuery(RAW_FLOAT_COL, 99900), 1},
+ {buildCountQuery(RAW_DOUBLE_COL, 99900), 1}
};
+ //@formatter:on
}
private static String buildCountQuery(String filterCol, Number min, Number max, boolean inclusive) {
if (inclusive) {
- return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " between "
- + buildFilter(filterCol, min, max);
+ return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " between " + buildFilter(filterCol,
+ min, max);
} else {
- return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " > "
- + formatValue(filterCol, min) + " and " + filterCol + " < " + formatValue(filterCol, max);
+ return "select count(*) from " + RAW_TABLE_NAME + " where " + filterCol + " > " + formatValue(filterCol, min)
+ + " and " + filterCol + " < " + formatValue(filterCol, max);
}
}
@@ -309,7 +353,6 @@ public class RangeQueriesTest extends BaseQueriesTest {
_indexSegment = immutableSegment;
_indexSegments = Arrays.asList(immutableSegment, immutableSegment);
-
Operator<?> operator = getOperator(query);
assertTrue(operator instanceof FastFilteredCountOperator);
List<Object> aggregationResult = ((FastFilteredCountOperator) operator).nextBlock().getResults();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
index 7b422b74d6..a21891e238 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
@@ -203,7 +203,7 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
private ImmutableRoaringBitmap queryRangeBitmap(long value, long columnMax) {
RangeBitmap rangeBitmap = mapRangeBitmap();
- if (Long.compareUnsigned(value, columnMax) < 0) {
+ if (Long.compareUnsigned(value, columnMax) <= 0) {
return rangeBitmap.eq(value).toMutableRoaringBitmap();
} else {
return new MutableRoaringBitmap();
@@ -230,7 +230,7 @@ public class BitSlicedRangeIndexReader implements RangeIndexReader<ImmutableRoar
private int queryRangeBitmapCardinality(long value, long columnMax) {
RangeBitmap rangeBitmap = mapRangeBitmap();
- if (Long.compareUnsigned(value, columnMax) < 0) {
+ if (Long.compareUnsigned(value, columnMax) <= 0) {
return (int) rangeBitmap.eqCardinality(value);
} else {
return 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org