You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2018/11/27 01:30:12 UTC
[incubator-pinot] branch master updated: Optimize all filter
predicates by adding isAlwaysTrue() (#3535)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8bf0050 Optimize all filter predicates by adding isAlwaysTrue() (#3535)
8bf0050 is described below
commit 8bf00503ccca60894593002b81e6b45afa4ee506
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Mon Nov 26 17:30:07 2018 -0800
Optimize all filter predicates by adding isAlwaysTrue() (#3535)
When filter predicate is always evaluated to true, we can optimize the filter operator by replacing it with MatchAllFilterOperator.
This is especially useful for range filters on time column which spans a very long time range.
Also added method isResultMatchingAll() in BaseFilterOperator for the optimization.
---
.../docidsets/BitmapBasedBlockDocIdSet.java | 41 ---------
.../core/operator/filter/AndFilterOperator.java | 25 +++---
.../core/operator/filter/BaseFilterOperator.java | 13 ++-
.../operator/filter/BitmapBasedFilterOperator.java | 15 ++--
.../core/operator/filter/EmptyFilterOperator.java | 8 +-
.../core/operator/filter/FilterOperatorUtils.java | 65 +++++++++++++-
.../operator/filter/MatchAllFilterOperator.java | 8 +-
.../core/operator/filter/OrFilterOperator.java | 25 +++---
.../operator/filter/ScanBasedFilterOperator.java | 16 ++--
.../SortedInvertedIndexBasedFilterOperator.java | 15 ++--
.../BaseDictionaryBasedPredicateEvaluator.java | 12 +++
.../BaseRawValueBasedPredicateEvaluator.java | 15 ++--
.../predicate/EqualsPredicateEvaluatorFactory.java | 9 +-
.../predicate/InPredicateEvaluatorFactory.java | 14 ++--
.../NotEqualsPredicateEvaluatorFactory.java | 9 +-
.../predicate/NotInPredicateEvaluatorFactory.java | 16 ++--
.../filter/predicate/PredicateEvaluator.java | 5 ++
.../predicate/RangePredicateEvaluatorFactory.java | 33 ++++----
.../RegexpLikePredicateEvaluatorFactory.java | 5 --
.../linkedin/pinot/core/plan/FilterPlanNode.java | 37 ++++----
.../startree/operator/StarTreeFilterOperator.java | 58 ++++++-------
.../operator/filter/AndFilterOperatorTest.java | 27 +++---
.../operator/filter/FilterOperatorUtilsTest.java | 98 ++++++++++++++++++++++
.../{ => core}/operator/filter/IntRangesTest.java | 3 +-
.../operator/filter/OrFilterOperatorTest.java | 20 ++---
.../core/operator/filter/TestFilterOperator.java | 78 +++++++++++++++++
...ngeOfflineDictionaryPredicateEvaluatorTest.java | 44 +++++++---
.../operator/filter/FilterOperatorTestUtils.java | 85 -------------------
...nerSegmentAggregationMultiValueQueriesTest.java | 12 +--
...InnerSegmentSelectionMultiValueQueriesTest.java | 6 +-
...terSegmentAggregationMultiValueQueriesTest.java | 32 +++----
...onaryAggregationPlanClusterIntegrationTest.java | 31 +------
.../com/linkedin/pinot/perf/RawIndexBenchmark.java | 7 +-
33 files changed, 514 insertions(+), 373 deletions(-)
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapBasedBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapBasedBlockDocIdSet.java
deleted file mode 100644
index 923cb61..0000000
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapBasedBlockDocIdSet.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.linkedin.pinot.core.operator.docidsets;
-
-import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-
-import com.linkedin.pinot.core.common.BlockDocIdIterator;
-import com.linkedin.pinot.core.common.BlockDocIdSet;
-import com.linkedin.pinot.core.operator.dociditerators.BitmapDocIdIterator;
-
-public final class BitmapBasedBlockDocIdSet implements BlockDocIdSet {
- private final ImmutableRoaringBitmap bitmap;
-
- public BitmapBasedBlockDocIdSet(ImmutableRoaringBitmap bitmap) {
- this.bitmap = bitmap;
- }
-
- @Override
- public BlockDocIdIterator iterator() {
- return new BitmapDocIdIterator(bitmap.getIntIterator());
- }
-
- @Override
- public <T> T getRaw() {
- return null;
- }
-
-}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/AndFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/AndFilterOperator.java
index 978e727..b29133c 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/AndFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/AndFilterOperator.java
@@ -15,6 +15,7 @@
*/
package com.linkedin.pinot.core.operator.filter;
+import com.google.common.base.Preconditions;
import com.linkedin.pinot.core.operator.blocks.FilterBlock;
import com.linkedin.pinot.core.operator.docidsets.AndBlockDocIdSet;
import com.linkedin.pinot.core.operator.docidsets.FilterBlockDocIdSet;
@@ -27,7 +28,19 @@ public class AndFilterOperator extends BaseFilterOperator {
private final List<BaseFilterOperator> _filterOperators;
- public AndFilterOperator(List<BaseFilterOperator> filterOperators) {
+ AndFilterOperator(List<BaseFilterOperator> filterOperators) {
+ // NOTE:
+ // EmptyFilterOperator and MatchAllFilterOperator should not be passed into the AndFilterOperator for performance
+ // concern.
+ // If there is any EmptyFilterOperator inside AndFilterOperator, the whole AndFilterOperator is equivalent to a
+ // EmptyFilterOperator; MatchAllFilterOperator should be ignored in AndFilterOperator.
+ // After removing the MatchAllFilterOperators, if there is no child filter operator left, use
+ // MatchAllFilterOperator; if there is only one child filter operator left, use the child filter operator directly.
+ // These checks should be performed before constructing the AndFilterOperator.
+ for (BaseFilterOperator filterOperator : filterOperators) {
+ Preconditions.checkArgument(!filterOperator.isResultEmpty() && !filterOperator.isResultMatchingAll());
+ }
+
_filterOperators = filterOperators;
}
@@ -41,16 +54,6 @@ public class AndFilterOperator extends BaseFilterOperator {
}
@Override
- public boolean isResultEmpty() {
- for (BaseFilterOperator filterOperator : _filterOperators) {
- if (filterOperator.isResultEmpty()) {
- return true;
- }
- }
- return false;
- }
-
- @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BaseFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BaseFilterOperator.java
index 6c0f287..d5b6021 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BaseFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BaseFilterOperator.java
@@ -25,7 +25,16 @@ import com.linkedin.pinot.core.operator.blocks.FilterBlock;
public abstract class BaseFilterOperator extends BaseOperator<FilterBlock> {
/**
- * Returns {@code true} if the result is always empty (without calling {@link #nextBlock()}), {@code false} otherwise.
+ * Returns {@code true} if the result is always empty, {@code false} otherwise.
*/
- public abstract boolean isResultEmpty();
+ public boolean isResultEmpty() {
+ return false;
+ }
+
+ /**
+ * Returns {@code true} if the result matches all the records, {@code false} otherwise.
+ */
+ public boolean isResultMatchingAll() {
+ return false;
+ }
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BitmapBasedFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BitmapBasedFilterOperator.java
index 55f0def..8de7ed8 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BitmapBasedFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/BitmapBasedFilterOperator.java
@@ -15,6 +15,7 @@
*/
package com.linkedin.pinot.core.operator.filter;
+import com.google.common.base.Preconditions;
import com.linkedin.pinot.core.common.DataSource;
import com.linkedin.pinot.core.operator.blocks.FilterBlock;
import com.linkedin.pinot.core.operator.docidsets.BitmapDocIdSet;
@@ -40,8 +41,15 @@ public class BitmapBasedFilterOperator extends BaseFilterOperator {
private final int _endDocId;
private final boolean _exclusive;
- public BitmapBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int startDocId,
+ BitmapBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int startDocId,
int endDocId) {
+ // NOTE:
+ // Predicate that is always evaluated as true or false should not be passed into the BitmapBasedFilterOperator for
+ // performance concern.
+ // If predicate is always evaluated as true, use MatchAllFilterOperator; if predicate is always evaluated as false,
+ // use EmptyFilterOperator.
+ Preconditions.checkArgument(!predicateEvaluator.isAlwaysTrue() && !predicateEvaluator.isAlwaysFalse());
+
_predicateEvaluator = predicateEvaluator;
_dataSource = dataSource;
_bitmaps = null;
@@ -91,11 +99,6 @@ public class BitmapBasedFilterOperator extends BaseFilterOperator {
}
@Override
- public boolean isResultEmpty() {
- return _predicateEvaluator != null && _predicateEvaluator.isAlwaysFalse();
- }
-
- @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/EmptyFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/EmptyFilterOperator.java
index 3068585..e60e0e3 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/EmptyFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/EmptyFilterOperator.java
@@ -34,13 +34,13 @@ public final class EmptyFilterOperator extends BaseFilterOperator {
}
@Override
- protected FilterBlock getNextBlock() {
- return EmptyFilterBlock.getInstance();
+ public final boolean isResultEmpty() {
+ return true;
}
@Override
- public boolean isResultEmpty() {
- return true;
+ protected FilterBlock getNextBlock() {
+ return EmptyFilterBlock.getInstance();
}
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtils.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtils.java
index a66d4c4..8f5b5a1 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtils.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -19,6 +19,7 @@ import com.linkedin.pinot.core.common.DataSource;
import com.linkedin.pinot.core.common.DataSourceMetadata;
import com.linkedin.pinot.core.common.Predicate;
import com.linkedin.pinot.core.operator.filter.predicate.PredicateEvaluator;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -37,11 +38,18 @@ public class FilterOperatorUtils {
* Returns the leaf filter operator (i.e. not {@link AndFilterOperator} or {@link OrFilterOperator}).
*/
public static BaseFilterOperator getLeafFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource,
- int startDocId, int endDocId) {
+ int numDocs) {
if (predicateEvaluator.isAlwaysFalse()) {
return EmptyFilterOperator.getInstance();
+ } else if (predicateEvaluator.isAlwaysTrue()) {
+ return new MatchAllFilterOperator(numDocs);
}
+ int startDocId = 0;
+ // NOTE: end document Id is inclusive
+ // TODO: make it exclusive
+ int endDocId = numDocs - 1;
+
// Use inverted index if the predicate type is not RANGE or REGEXP_LIKE for efficiency
DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata();
Predicate.Type predicateType = predicateEvaluator.getPredicateType();
@@ -58,12 +66,65 @@ public class FilterOperatorUtils {
}
/**
+ * Returns the AND filter operator or equivalent filter operator.
+ */
+ public static BaseFilterOperator getAndFilterOperator(List<BaseFilterOperator> filterOperators, int numDocs,
+ @Nullable Map<String, String> debugOptions) {
+ List<BaseFilterOperator> childFilterOperators = new ArrayList<>(filterOperators.size());
+ for (BaseFilterOperator filterOperator : filterOperators) {
+ if (filterOperator.isResultEmpty()) {
+ return EmptyFilterOperator.getInstance();
+ } else if (!filterOperator.isResultMatchingAll()) {
+ childFilterOperators.add(filterOperator);
+ }
+ }
+ int numChildFilterOperators = childFilterOperators.size();
+ if (numChildFilterOperators == 0) {
+ // Return match all filter operator if all child filter operators match all records
+ return new MatchAllFilterOperator(numDocs);
+ } else if (numChildFilterOperators == 1) {
+ // Return the child filter operator if only one left
+ return childFilterOperators.get(0);
+ } else {
+ // Return the AND filter operator with re-ordered child filter operators
+ FilterOperatorUtils.reorderAndFilterChildOperators(childFilterOperators, debugOptions);
+ return new AndFilterOperator(childFilterOperators);
+ }
+ }
+
+ /**
+ * Returns the OR filter operator or equivalent filter operator.
+ */
+ public static BaseFilterOperator getOrFilterOperator(List<BaseFilterOperator> filterOperators, int numDocs,
+ @Nullable Map<String, String> debugOptions) {
+ List<BaseFilterOperator> childFilterOperators = new ArrayList<>(filterOperators.size());
+ for (BaseFilterOperator filterOperator : filterOperators) {
+ if (filterOperator.isResultMatchingAll()) {
+ return new MatchAllFilterOperator(numDocs);
+ } else if (!filterOperator.isResultEmpty()) {
+ childFilterOperators.add(filterOperator);
+ }
+ }
+ int numChildFilterOperators = childFilterOperators.size();
+ if (numChildFilterOperators == 0) {
+ // Return empty filter operator if all child filter operators's result is empty
+ return EmptyFilterOperator.getInstance();
+ } else if (numChildFilterOperators == 1) {
+ // Return the child filter operator if only one left
+ return childFilterOperators.get(0);
+ } else {
+ // Return the OR filter operator with child filter operators
+ return new OrFilterOperator(childFilterOperators);
+ }
+ }
+
+ /**
* For AND filter operator, reorders its child filter operators based on the their cost and puts the ones with
* inverted index first in order to reduce the number of documents to be processed.
* <p>Special filter operators such as {@link MatchAllFilterOperator} and {@link EmptyFilterOperator} should be
* removed from the list before calling this method.
*/
- public static void reorderAndFilterChildOperators(List<BaseFilterOperator> filterOperators,
+ private static void reorderAndFilterChildOperators(List<BaseFilterOperator> filterOperators,
@Nullable Map<String, String> debugOptions) {
filterOperators.sort(new Comparator<BaseFilterOperator>() {
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/MatchAllFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/MatchAllFilterOperator.java
index 89e24fd..4de4faf 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/MatchAllFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/MatchAllFilterOperator.java
@@ -29,13 +29,13 @@ public class MatchAllFilterOperator extends BaseFilterOperator {
}
@Override
- protected FilterBlock getNextBlock() {
- return new FilterBlock(new SizeBasedDocIdSet(_maxDocId));
+ public final boolean isResultMatchingAll() {
+ return true;
}
@Override
- public boolean isResultEmpty() {
- return false;
+ protected FilterBlock getNextBlock() {
+ return new FilterBlock(new SizeBasedDocIdSet(_maxDocId));
}
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/OrFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/OrFilterOperator.java
index 3df7ba6..38d9f25 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/OrFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/OrFilterOperator.java
@@ -15,6 +15,7 @@
*/
package com.linkedin.pinot.core.operator.filter;
+import com.google.common.base.Preconditions;
import com.linkedin.pinot.core.operator.blocks.FilterBlock;
import com.linkedin.pinot.core.operator.docidsets.FilterBlockDocIdSet;
import com.linkedin.pinot.core.operator.docidsets.OrBlockDocIdSet;
@@ -27,7 +28,19 @@ public class OrFilterOperator extends BaseFilterOperator {
private List<BaseFilterOperator> _filterOperators;
- public OrFilterOperator(List<BaseFilterOperator> filterOperators) {
+ OrFilterOperator(List<BaseFilterOperator> filterOperators) {
+ // NOTE:
+ // EmptyFilterOperator and MatchAllFilterOperator should not be passed into the OrFilterOperator for performance
+ // concern.
+ // If there is any MatchAllFilterOperator inside OrFilterOperator, the whole OrFilterOperator is equivalent to a
+ // MatchAllFilterOperator; EmptyFilterOperator should be ignored in OrFilterOperator.
+ // After removing the EmptyFilterOperator, if there is no child filter operator left, use EmptyFilterOperator;
+ // if there is only one child filter operator left, use the child filter operator directly.
+ // These checks should be performed before constructing the OrFilterOperator.
+ for (BaseFilterOperator filterOperator : filterOperators) {
+ Preconditions.checkArgument(!filterOperator.isResultEmpty() && !filterOperator.isResultMatchingAll());
+ }
+
_filterOperators = filterOperators;
}
@@ -41,16 +54,6 @@ public class OrFilterOperator extends BaseFilterOperator {
}
@Override
- public boolean isResultEmpty() {
- for (BaseFilterOperator filterOperator : _filterOperators) {
- if (!filterOperator.isResultEmpty()) {
- return false;
- }
- }
- return true;
- }
-
- @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/ScanBasedFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/ScanBasedFilterOperator.java
index 389ad50..94761f8 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/ScanBasedFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/ScanBasedFilterOperator.java
@@ -15,6 +15,7 @@
*/
package com.linkedin.pinot.core.operator.filter;
+import com.google.common.base.Preconditions;
import com.linkedin.pinot.core.common.Block;
import com.linkedin.pinot.core.common.BlockMetadata;
import com.linkedin.pinot.core.common.BlockValSet;
@@ -37,8 +38,14 @@ public class ScanBasedFilterOperator extends BaseFilterOperator {
// Inclusive
private final int _endDocId;
- public ScanBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int startDocId,
- int endDocId) {
+ ScanBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource, int startDocId, int endDocId) {
+ // NOTE:
+ // Predicate that is always evaluated as true or false should not be passed into the ScanBasedFilterOperator for
+ // performance concern.
+ // If predicate is always evaluated as true, use MatchAllFilterOperator; if predicate is always evaluated as false,
+ // use EmptyFilterOperator.
+ Preconditions.checkArgument(!predicateEvaluator.isAlwaysTrue() && !predicateEvaluator.isAlwaysFalse());
+
_predicateEvaluator = predicateEvaluator;
_dataSource = dataSource;
_startDocId = startDocId;
@@ -68,11 +75,6 @@ public class ScanBasedFilterOperator extends BaseFilterOperator {
}
@Override
- public boolean isResultEmpty() {
- return _predicateEvaluator.isAlwaysFalse();
- }
-
- @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/SortedInvertedIndexBasedFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/SortedInvertedIndexBasedFilterOperator.java
index 912b0ef..5e0fc11 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/SortedInvertedIndexBasedFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/SortedInvertedIndexBasedFilterOperator.java
@@ -15,6 +15,7 @@
*/
package com.linkedin.pinot.core.operator.filter;
+import com.google.common.base.Preconditions;
import com.linkedin.pinot.common.utils.Pairs.IntPair;
import com.linkedin.pinot.core.common.DataSource;
import com.linkedin.pinot.core.io.reader.impl.v1.SortedIndexReader;
@@ -36,8 +37,15 @@ public class SortedInvertedIndexBasedFilterOperator extends BaseFilterOperator {
// Inclusive
private final int _endDocId;
- public SortedInvertedIndexBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource,
+ SortedInvertedIndexBasedFilterOperator(PredicateEvaluator predicateEvaluator, DataSource dataSource,
int startDocId, int endDocId) {
+ // NOTE:
+ // Predicate that is always evaluated as true or false should not be passed into the
+ // SortedInvertedIndexBasedFilterOperator for performance concern.
+ // If predicate is always evaluated as true, use MatchAllFilterOperator; if predicate is always evaluated as false,
+ // use EmptyFilterOperator.
+ Preconditions.checkArgument(!predicateEvaluator.isAlwaysTrue() && !predicateEvaluator.isAlwaysFalse());
+
_predicateEvaluator = predicateEvaluator;
_dataSource = dataSource;
_startDocId = startDocId;
@@ -147,11 +155,6 @@ public class SortedInvertedIndexBasedFilterOperator extends BaseFilterOperator {
}
@Override
- public boolean isResultEmpty() {
- return _predicateEvaluator.isAlwaysFalse();
- }
-
- @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
index 08835a8..0d3de77 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
@@ -16,6 +16,18 @@
package com.linkedin.pinot.core.operator.filter.predicate;
public abstract class BaseDictionaryBasedPredicateEvaluator extends BasePredicateEvaluator {
+ protected boolean _alwaysTrue;
+ protected boolean _alwaysFalse;
+
+ @Override
+ public boolean isAlwaysTrue() {
+ return _alwaysTrue;
+ }
+
+ @Override
+ public boolean isAlwaysFalse() {
+ return _alwaysFalse;
+ }
@Override
public final boolean isDictionaryBased() {
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
index f32df69..858ec0b 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
@@ -23,18 +23,23 @@ public abstract class BaseRawValueBasedPredicateEvaluator extends BasePredicateE
}
@Override
- public final int[] getMatchingDictIds() {
- throw new UnsupportedOperationException();
+ public final boolean isAlwaysTrue() {
+ return false;
}
@Override
- public final int[] getNonMatchingDictIds() {
+ public final boolean isAlwaysFalse() {
+ return false;
+ }
+
+ @Override
+ public final int[] getMatchingDictIds() {
throw new UnsupportedOperationException();
}
@Override
- public boolean isAlwaysFalse() {
- return false;
+ public final int[] getNonMatchingDictIds() {
+ throw new UnsupportedOperationException();
}
/**
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
index 5ed998b..38f46d7 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
@@ -73,8 +73,12 @@ public class EqualsPredicateEvaluatorFactory {
_matchingDictId = dictionary.indexOf(eqPredicate.getEqualsValue());
if (_matchingDictId >= 0) {
_matchingDictIds = new int[]{_matchingDictId};
+ if (dictionary.length() == 1) {
+ _alwaysTrue = true;
+ }
} else {
_matchingDictIds = new int[0];
+ _alwaysFalse = true;
}
}
@@ -84,11 +88,6 @@ public class EqualsPredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return _matchingDictId < 0;
- }
-
- @Override
public boolean applySV(int dictId) {
return _matchingDictId == dictId;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
index f99329e..1d399b3 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
@@ -79,6 +79,7 @@ public class InPredicateEvaluatorFactory {
private static final class DictionaryBasedInPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator {
final IntSet _matchingDictIdSet;
+ final int _numMatchingDictIds;
int[] _matchingDictIds;
DictionaryBasedInPredicateEvaluator(InPredicate inPredicate, Dictionary dictionary) {
@@ -90,6 +91,12 @@ public class InPredicateEvaluatorFactory {
_matchingDictIdSet.add(dictId);
}
}
+ _numMatchingDictIds = _matchingDictIdSet.size();
+ if (_numMatchingDictIds == 0) {
+ _alwaysFalse = true;
+ } else if (dictionary.length() == _numMatchingDictIds) {
+ _alwaysTrue = true;
+ }
}
@Override
@@ -98,18 +105,13 @@ public class InPredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return _matchingDictIdSet.isEmpty();
- }
-
- @Override
public boolean applySV(int dictId) {
return _matchingDictIdSet.contains(dictId);
}
@Override
public int getNumMatchingDictIds() {
- return _matchingDictIdSet.size();
+ return _numMatchingDictIds;
}
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
index ce27adb..e2deef5 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
@@ -75,8 +75,12 @@ public class NotEqualsPredicateEvaluatorFactory {
_nonMatchingDictId = dictionary.indexOf(nEqPredicate.getNotEqualsValue());
if (_nonMatchingDictId >= 0) {
_nonMatchingDictIds = new int[]{_nonMatchingDictId};
+ if (dictionary.length() == 1) {
+ _alwaysFalse = true;
+ }
} else {
_nonMatchingDictIds = new int[0];
+ _alwaysTrue = true;
}
_dictionary = dictionary;
}
@@ -87,11 +91,6 @@ public class NotEqualsPredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return _nonMatchingDictIds.length == _dictionary.length();
- }
-
- @Override
public boolean applySV(int dictId) {
return _nonMatchingDictId != dictId;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
index b16b85c..4d55aee 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
@@ -79,6 +79,7 @@ public class NotInPredicateEvaluatorFactory {
public static final class DictionaryBasedNotInPredicateEvaluator extends BaseDictionaryBasedPredicateEvaluator {
final IntSet _nonMatchingDictIdSet;
+ final int _numNonMatchingDictIds;
final Dictionary _dictionary;
int[] _matchingDictIds;
int[] _nonMatchingDictIds;
@@ -92,6 +93,12 @@ public class NotInPredicateEvaluatorFactory {
_nonMatchingDictIdSet.add(dictId);
}
}
+ _numNonMatchingDictIds = _nonMatchingDictIdSet.size();
+ if (_numNonMatchingDictIds == 0) {
+ _alwaysTrue = true;
+ } else if (dictionary.length() == _numNonMatchingDictIds) {
+ _alwaysFalse = true;
+ }
_dictionary = dictionary;
}
@@ -101,11 +108,6 @@ public class NotInPredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return _nonMatchingDictIdSet.size() == _dictionary.length();
- }
-
- @Override
public boolean applySV(int dictId) {
return !_nonMatchingDictIdSet.contains(dictId);
}
@@ -114,7 +116,7 @@ public class NotInPredicateEvaluatorFactory {
public int[] getMatchingDictIds() {
if (_matchingDictIds == null) {
int dictionarySize = _dictionary.length();
- _matchingDictIds = new int[dictionarySize - _nonMatchingDictIdSet.size()];
+ _matchingDictIds = new int[dictionarySize - _numNonMatchingDictIds];
int index = 0;
for (int dictId = 0; dictId < dictionarySize; dictId++) {
if (!_nonMatchingDictIdSet.contains(dictId)) {
@@ -127,7 +129,7 @@ public class NotInPredicateEvaluatorFactory {
@Override
public int getNumNonMatchingDictIds() {
- return _nonMatchingDictIdSet.size();
+ return _numNonMatchingDictIds;
}
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/PredicateEvaluator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/PredicateEvaluator.java
index 2385cee..7205e37 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/PredicateEvaluator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/PredicateEvaluator.java
@@ -39,6 +39,11 @@ public interface PredicateEvaluator {
boolean isExclusive();
/**
+ * Return whether the predicate will always be evaluated as true.
+ */
+ boolean isAlwaysTrue();
+
+ /**
* Return whether the predicate will always be evaluated as false.
*/
boolean isAlwaysFalse();
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
index 6184e20..b688563 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -115,7 +115,13 @@ public class RangePredicateEvaluatorFactory {
}
}
}
+
_numMatchingDictIds = _endDictId - _startDictId;
+ if (_numMatchingDictIds <= 0) {
+ _alwaysFalse = true;
+ } else if (dictionary.length() == _numMatchingDictIds) {
+ _alwaysTrue = true;
+ }
}
@Override
@@ -124,11 +130,6 @@ public class RangePredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return _startDictId >= _endDictId;
- }
-
- @Override
public boolean applySV(int dictId) {
return _startDictId <= dictId && _endDictId > dictId;
}
@@ -157,6 +158,7 @@ public class RangePredicateEvaluatorFactory {
private static final class RealtimeDictionaryBasedRangePredicateEvaluator
extends BaseDictionaryBasedPredicateEvaluator {
final IntSet _matchingDictIdSet;
+ final int _numMatchingDictIds;
int[] _matchingDictIds;
RealtimeDictionaryBasedRangePredicateEvaluator(RangePredicate rangePredicate, MutableDictionary dictionary) {
@@ -164,6 +166,8 @@ public class RangePredicateEvaluatorFactory {
int dictionarySize = dictionary.length();
if (dictionarySize == 0) {
+ _numMatchingDictIds = 0;
+ _alwaysFalse = true;
return;
}
@@ -184,6 +188,13 @@ public class RangePredicateEvaluatorFactory {
_matchingDictIdSet.add(dictId);
}
}
+
+ _numMatchingDictIds = _matchingDictIdSet.size();
+ if (_numMatchingDictIds == 0) {
+ _alwaysFalse = true;
+ } else if (dictionarySize == _numMatchingDictIds) {
+ _alwaysTrue = true;
+ }
}
@Override
@@ -198,7 +209,7 @@ public class RangePredicateEvaluatorFactory {
@Override
public int getNumMatchingDictIds() {
- return _matchingDictIdSet.size();
+ return _numMatchingDictIds;
}
@Override
@@ -208,16 +219,6 @@ public class RangePredicateEvaluatorFactory {
}
return _matchingDictIds;
}
-
- @Override
- public int[] getNonMatchingDictIds() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isAlwaysFalse() {
- return _matchingDictIdSet.isEmpty();
- }
}
private static final class IntRawValueBasedRangePredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
index d6ac01a..037c40f 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
@@ -75,11 +75,6 @@ public class RegexpLikePredicateEvaluatorFactory {
}
@Override
- public boolean isAlwaysFalse() {
- return false;
- }
-
- @Override
public boolean applySV(int dictId) {
return _pattern.matcher(_dictionary.getStringValue(dictId)).find();
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/plan/FilterPlanNode.java b/pinot-core/src/main/java/com/linkedin/pinot/core/plan/FilterPlanNode.java
index 226f1fc..797a589 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/plan/FilterPlanNode.java
@@ -22,12 +22,10 @@ import com.linkedin.pinot.common.utils.request.RequestUtils;
import com.linkedin.pinot.core.common.DataSource;
import com.linkedin.pinot.core.common.Predicate;
import com.linkedin.pinot.core.indexsegment.IndexSegment;
-import com.linkedin.pinot.core.operator.filter.AndFilterOperator;
import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
import com.linkedin.pinot.core.operator.filter.EmptyFilterOperator;
import com.linkedin.pinot.core.operator.filter.FilterOperatorUtils;
import com.linkedin.pinot.core.operator.filter.MatchAllFilterOperator;
-import com.linkedin.pinot.core.operator.filter.OrFilterOperator;
import com.linkedin.pinot.core.operator.filter.predicate.PredicateEvaluator;
import com.linkedin.pinot.core.operator.filter.predicate.PredicateEvaluatorProvider;
import java.util.ArrayList;
@@ -59,49 +57,50 @@ public class FilterPlanNode implements PlanNode {
*/
private static BaseFilterOperator constructPhysicalOperator(FilterQueryTree filterQueryTree, IndexSegment segment,
@Nullable Map<String, String> debugOptions) {
+ int numDocs = segment.getSegmentMetadata().getTotalRawDocs();
if (filterQueryTree == null) {
- return new MatchAllFilterOperator(segment.getSegmentMetadata().getTotalRawDocs());
+ return new MatchAllFilterOperator(numDocs);
}
// For non-leaf node, recursively create the child filter operators
FilterOperator filterType = filterQueryTree.getOperator();
if (filterType == FilterOperator.AND || filterType == FilterOperator.OR) {
+ // Non-leaf filter operator
List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
List<BaseFilterOperator> childFilterOperators = new ArrayList<>(childFilters.size());
if (filterType == FilterOperator.AND) {
+ // AND operator
for (FilterQueryTree childFilter : childFilters) {
BaseFilterOperator childFilterOperator = constructPhysicalOperator(childFilter, segment, debugOptions);
if (childFilterOperator.isResultEmpty()) {
+ // Return empty filter operator if any of the child filter operator's result is empty
return EmptyFilterOperator.getInstance();
+ } else if (!childFilterOperator.isResultMatchingAll()) {
+ // Remove child filter operators that match all records
+ childFilterOperators.add(childFilterOperator);
}
- childFilterOperators.add(childFilterOperator);
}
- FilterOperatorUtils.reorderAndFilterChildOperators(childFilterOperators, debugOptions);
- return new AndFilterOperator(childFilterOperators);
+ return FilterOperatorUtils.getAndFilterOperator(childFilterOperators, numDocs, debugOptions);
} else {
+ // OR operator
for (FilterQueryTree childFilter : childFilters) {
BaseFilterOperator childFilterOperator = constructPhysicalOperator(childFilter, segment, debugOptions);
- if (!childFilterOperator.isResultEmpty()) {
+ if (childFilterOperator.isResultMatchingAll()) {
+ // Return match all filter operator if any of the child filter operator matches all records
+ return new MatchAllFilterOperator(numDocs);
+ } else if (!childFilterOperator.isResultEmpty()) {
+ // Remove child filter operators whose result is empty
childFilterOperators.add(childFilterOperator);
}
}
- if (childFilterOperators.isEmpty()) {
- return EmptyFilterOperator.getInstance();
- } else if (childFilterOperators.size() == 1) {
- return childFilterOperators.get(0);
- } else {
- return new OrFilterOperator(childFilterOperators);
- }
+ return FilterOperatorUtils.getOrFilterOperator(childFilterOperators, numDocs, debugOptions);
}
} else {
+ // Leaf filter operator
Predicate predicate = Predicate.newPredicate(filterQueryTree);
DataSource dataSource = segment.getDataSource(filterQueryTree.getColumn());
PredicateEvaluator predicateEvaluator = PredicateEvaluatorProvider.getPredicateEvaluator(predicate, dataSource);
- int startDocId = 0;
- // TODO: make it exclusive
- // NOTE: end is inclusive
- int endDocId = segment.getSegmentMetadata().getTotalRawDocs() - 1;
- return FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource, startDocId, endDocId);
+ return FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource, numDocs);
}
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/startree/operator/StarTreeFilterOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/startree/operator/StarTreeFilterOperator.java
index 3e008b8..c72756a 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/startree/operator/StarTreeFilterOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/startree/operator/StarTreeFilterOperator.java
@@ -20,9 +20,9 @@ import com.linkedin.pinot.core.common.DataSource;
import com.linkedin.pinot.core.common.Predicate;
import com.linkedin.pinot.core.operator.blocks.EmptyFilterBlock;
import com.linkedin.pinot.core.operator.blocks.FilterBlock;
-import com.linkedin.pinot.core.operator.filter.AndFilterOperator;
import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
import com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator;
+import com.linkedin.pinot.core.operator.filter.EmptyFilterOperator;
import com.linkedin.pinot.core.operator.filter.FilterOperatorUtils;
import com.linkedin.pinot.core.operator.filter.predicate.PredicateEvaluator;
import com.linkedin.pinot.core.operator.filter.predicate.PredicateEvaluatorProvider;
@@ -137,21 +137,17 @@ public class StarTreeFilterOperator extends BaseFilterOperator {
_debugOptions = debugOptions;
if (rootFilterNode != null) {
+ _predicateEvaluatorsMap = new HashMap<>();
+ _matchingDictIdsMap = new HashMap<>();
+
// Process the filter tree and get a map from column to a list of predicates applied to it
Map<String, List<Predicate>> predicatesMap = getPredicatesMap(rootFilterNode);
- // Remove columns with predicates from group-by columns because we won't use star node for that column
- _groupByColumns.removeAll(predicatesMap.keySet());
-
- int numColumnsInPredicates = predicatesMap.size();
- _predicateEvaluatorsMap = new HashMap<>(numColumnsInPredicates);
- _matchingDictIdsMap = new HashMap<>(numColumnsInPredicates);
-
// Initialize the predicate evaluators map
for (Map.Entry<String, List<Predicate>> entry : predicatesMap.entrySet()) {
String columnName = entry.getKey();
List<Predicate> predicates = entry.getValue();
- List<PredicateEvaluator> predicateEvaluators = new ArrayList<>(predicates.size());
+ List<PredicateEvaluator> predicateEvaluators = new ArrayList<>();
DataSource dataSource = starTreeV2.getDataSource(columnName);
for (Predicate predicate : predicates) {
@@ -161,11 +157,17 @@ public class StarTreeFilterOperator extends BaseFilterOperator {
if (predicateEvaluator.isAlwaysFalse()) {
_resultEmpty = true;
return;
+ } else if (!predicateEvaluator.isAlwaysTrue()) {
+ predicateEvaluators.add(predicateEvaluator);
}
- predicateEvaluators.add(predicateEvaluator);
}
- _predicateEvaluatorsMap.put(columnName, predicateEvaluators);
+ if (!predicateEvaluators.isEmpty()) {
+ _predicateEvaluatorsMap.put(columnName, predicateEvaluators);
+ }
}
+
+ // Remove columns with predicates from group-by columns because we won't use star node for that column
+ _groupByColumns.removeAll(_predicateEvaluatorsMap.keySet());
} else {
_predicateEvaluatorsMap = Collections.emptyMap();
_matchingDictIdsMap = Collections.emptyMap();
@@ -200,16 +202,7 @@ public class StarTreeFilterOperator extends BaseFilterOperator {
if (_resultEmpty) {
return EmptyFilterBlock.getInstance();
}
- List<BaseFilterOperator> childFilterOperators = getChildFilterOperators();
- int numChildFilterOperators = childFilterOperators.size();
- if (numChildFilterOperators == 0) {
- return EmptyFilterBlock.getInstance();
- } else if (numChildFilterOperators == 1) {
- return childFilterOperators.get(0).nextBlock();
- } else {
- FilterOperatorUtils.reorderAndFilterChildOperators(childFilterOperators, _debugOptions);
- return new AndFilterOperator(childFilterOperators).nextBlock();
- }
+ return getFilterOperator().nextBlock();
}
@Override
@@ -218,35 +211,37 @@ public class StarTreeFilterOperator extends BaseFilterOperator {
}
@Override
+ public boolean isResultMatchingAll() {
+ return false;
+ }
+
+ @Override
public String getOperatorName() {
return OPERATOR_NAME;
}
/**
- * Helper method to get a list of child filter operators that match the matchingDictIdsMap.
+ * Helper method to get a filter operator that match the matchingDictIdsMap.
* <ul>
* <li>First go over the star tree and try to match as many columns as possible</li>
* <li>For the remaining columns, use other indexes to match them</li>
* </ul>
*/
- private List<BaseFilterOperator> getChildFilterOperators() {
+ private BaseFilterOperator getFilterOperator() {
StarTreeResult starTreeResult = traverseStarTree();
// If star tree result is null, the result for the filter operator will be empty, early terminate
if (starTreeResult == null) {
- return Collections.emptyList();
+ return EmptyFilterOperator.getInstance();
}
+ int numDocs = _starTreeV2.getMetadata().getNumDocs();
List<BaseFilterOperator> childFilterOperators =
new ArrayList<>(1 + starTreeResult._remainingPredicateColumns.size());
- int startDocId = 0;
- // Inclusive end document id
- int endDocId = _starTreeV2.getMetadata().getNumDocs() - 1;
-
// Add the bitmap of matching documents from star tree
childFilterOperators.add(
- new BitmapBasedFilterOperator(new ImmutableRoaringBitmap[]{starTreeResult._matchedDocIds}, startDocId, endDocId,
+ new BitmapBasedFilterOperator(new ImmutableRoaringBitmap[]{starTreeResult._matchedDocIds}, 0, numDocs - 1,
false));
// Add remaining predicates
@@ -254,12 +249,11 @@ public class StarTreeFilterOperator extends BaseFilterOperator {
List<PredicateEvaluator> predicateEvaluators = _predicateEvaluatorsMap.get(remainingPredicateColumn);
DataSource dataSource = _starTreeV2.getDataSource(remainingPredicateColumn);
for (PredicateEvaluator predicateEvaluator : predicateEvaluators) {
- childFilterOperators.add(
- FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource, startDocId, endDocId));
+ childFilterOperators.add(FilterOperatorUtils.getLeafFilterOperator(predicateEvaluator, dataSource, numDocs));
}
}
- return childFilterOperators;
+ return FilterOperatorUtils.getAndFilterOperator(childFilterOperators, numDocs, _debugOptions);
}
/**
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/AndFilterOperatorTest.java b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/AndFilterOperatorTest.java
similarity index 77%
rename from pinot-core/src/test/java/com/linkedin/pinot/operator/filter/AndFilterOperatorTest.java
rename to pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/AndFilterOperatorTest.java
index e497d4a..ef049fc 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/AndFilterOperatorTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/AndFilterOperatorTest.java
@@ -13,13 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.linkedin.pinot.operator.filter;
+package com.linkedin.pinot.core.operator.filter;
import com.linkedin.pinot.core.common.BlockDocIdIterator;
import com.linkedin.pinot.core.common.Constants;
-import com.linkedin.pinot.core.operator.filter.AndFilterOperator;
-import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
-import com.linkedin.pinot.core.operator.filter.OrFilterOperator;
import java.util.ArrayList;
import java.util.List;
import org.testng.Assert;
@@ -34,8 +31,8 @@ public class AndFilterOperatorTest {
int[] docIds2 = new int[]{3, 6, 8, 20, 28};
List<BaseFilterOperator> operators = new ArrayList<>();
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
+ operators.add(new TestFilterOperator(docIds1));
+ operators.add(new TestFilterOperator(docIds2));
AndFilterOperator andOperator = new AndFilterOperator(operators);
BlockDocIdIterator iterator = andOperator.nextBlock().getBlockDocIdSet().iterator();
@@ -51,9 +48,9 @@ public class AndFilterOperatorTest {
int[] docIds3 = new int[]{1, 2, 3, 6, 30};
List<BaseFilterOperator> operators = new ArrayList<>();
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds3));
+ operators.add(new TestFilterOperator(docIds1));
+ operators.add(new TestFilterOperator(docIds2));
+ operators.add(new TestFilterOperator(docIds3));
AndFilterOperator andOperator = new AndFilterOperator(operators);
BlockDocIdIterator iterator = andOperator.nextBlock().getBlockDocIdSet().iterator();
@@ -69,13 +66,13 @@ public class AndFilterOperatorTest {
int[] docIds3 = new int[]{1, 2, 3, 6, 30};
List<BaseFilterOperator> childOperators = new ArrayList<>();
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
+ childOperators.add(new TestFilterOperator(docIds1));
+ childOperators.add(new TestFilterOperator(docIds2));
AndFilterOperator childAndOperator = new AndFilterOperator(childOperators);
List<BaseFilterOperator> operators = new ArrayList<>();
operators.add(childAndOperator);
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds3));
+ operators.add(new TestFilterOperator(docIds3));
AndFilterOperator andOperator = new AndFilterOperator(operators);
BlockDocIdIterator iterator = andOperator.nextBlock().getBlockDocIdSet().iterator();
@@ -91,13 +88,13 @@ public class AndFilterOperatorTest {
int[] docIds3 = new int[]{1, 2, 3, 6, 30};
List<BaseFilterOperator> childOperators = new ArrayList<>();
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds3));
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
+ childOperators.add(new TestFilterOperator(docIds3));
+ childOperators.add(new TestFilterOperator(docIds2));
OrFilterOperator childOrOperator = new OrFilterOperator(childOperators);
List<BaseFilterOperator> operators = new ArrayList<>();
operators.add(childOrOperator);
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
+ operators.add(new TestFilterOperator(docIds1));
AndFilterOperator andOperator = new AndFilterOperator(operators);
BlockDocIdIterator iterator = andOperator.nextBlock().getBlockDocIdSet().iterator();
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtilsTest.java b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtilsTest.java
new file mode 100644
index 0000000..3d101a8
--- /dev/null
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/FilterOperatorUtilsTest.java
@@ -0,0 +1,98 @@
+/**
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.pinot.core.operator.filter;
+
+import java.util.Arrays;
+import java.util.Collections;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+public class FilterOperatorUtilsTest {
+ private static final int NUM_DOCS = 10;
+ private static final BaseFilterOperator EMPTY_FILTER_OPERATOR = EmptyFilterOperator.getInstance();
+ private static final BaseFilterOperator MATCH_ALL_FILTER_OPERATOR = new MatchAllFilterOperator(NUM_DOCS);
+ private static final BaseFilterOperator REGULAR_FILTER_OPERATOR = new TestFilterOperator(new int[]{1, 4, 7});
+
+ @Test
+ public void testGetAndFilterOperator() {
+ BaseFilterOperator filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Collections.emptyList(), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof MatchAllFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Collections.singletonList(EMPTY_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof EmptyFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Collections.singletonList(MATCH_ALL_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof MatchAllFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Collections.singletonList(REGULAR_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof TestFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Arrays.asList(EMPTY_FILTER_OPERATOR, MATCH_ALL_FILTER_OPERATOR),
+ NUM_DOCS, null);
+ assertTrue(filterOperator instanceof EmptyFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Arrays.asList(EMPTY_FILTER_OPERATOR, REGULAR_FILTER_OPERATOR),
+ NUM_DOCS, null);
+ assertTrue(filterOperator instanceof EmptyFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getAndFilterOperator(Arrays.asList(MATCH_ALL_FILTER_OPERATOR, REGULAR_FILTER_OPERATOR),
+ NUM_DOCS, null);
+ assertTrue(filterOperator instanceof TestFilterOperator);
+ }
+
+ @Test
+ public void testGetOrFilterOperator() {
+ BaseFilterOperator filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Collections.emptyList(), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof EmptyFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Collections.singletonList(EMPTY_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof EmptyFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Collections.singletonList(MATCH_ALL_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof MatchAllFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Collections.singletonList(REGULAR_FILTER_OPERATOR), NUM_DOCS, null);
+ assertTrue(filterOperator instanceof TestFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Arrays.asList(EMPTY_FILTER_OPERATOR, MATCH_ALL_FILTER_OPERATOR),
+ NUM_DOCS, null);
+ assertTrue(filterOperator instanceof MatchAllFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Arrays.asList(EMPTY_FILTER_OPERATOR, REGULAR_FILTER_OPERATOR), NUM_DOCS,
+ null);
+ assertTrue(filterOperator instanceof TestFilterOperator);
+
+ filterOperator =
+ FilterOperatorUtils.getOrFilterOperator(Arrays.asList(MATCH_ALL_FILTER_OPERATOR, REGULAR_FILTER_OPERATOR),
+ NUM_DOCS, null);
+ assertTrue(filterOperator instanceof MatchAllFilterOperator);
+ }
+}
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/IntRangesTest.java b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/IntRangesTest.java
similarity index 96%
rename from pinot-core/src/test/java/com/linkedin/pinot/operator/filter/IntRangesTest.java
rename to pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/IntRangesTest.java
index 1492983..23ea8f7 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/IntRangesTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/IntRangesTest.java
@@ -13,10 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.linkedin.pinot.operator.filter;
+package com.linkedin.pinot.core.operator.filter;
import com.linkedin.pinot.common.utils.Pairs;
-import com.linkedin.pinot.core.operator.filter.IntRanges;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/OrFilterOperatorTest.java b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/OrFilterOperatorTest.java
similarity index 82%
rename from pinot-core/src/test/java/com/linkedin/pinot/operator/filter/OrFilterOperatorTest.java
rename to pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/OrFilterOperatorTest.java
index cd5983d..9a64b62 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/OrFilterOperatorTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/OrFilterOperatorTest.java
@@ -13,12 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.linkedin.pinot.operator.filter;
+package com.linkedin.pinot.core.operator.filter;
import com.linkedin.pinot.core.common.BlockDocIdIterator;
import com.linkedin.pinot.core.common.Constants;
-import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
-import com.linkedin.pinot.core.operator.filter.OrFilterOperator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -41,8 +39,8 @@ public class OrFilterOperatorTest {
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> operators = new ArrayList<>();
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
+ operators.add(new TestFilterOperator(docIds1));
+ operators.add(new TestFilterOperator(docIds2));
OrFilterOperator orOperator = new OrFilterOperator(operators);
BlockDocIdIterator iterator = orOperator.nextBlock().getBlockDocIdSet().iterator();
@@ -64,9 +62,9 @@ public class OrFilterOperatorTest {
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> operators = new ArrayList<>();
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds3));
+ operators.add(new TestFilterOperator(docIds1));
+ operators.add(new TestFilterOperator(docIds2));
+ operators.add(new TestFilterOperator(docIds3));
OrFilterOperator orOperator = new OrFilterOperator(operators);
BlockDocIdIterator iterator = orOperator.nextBlock().getBlockDocIdSet().iterator();
@@ -88,13 +86,13 @@ public class OrFilterOperatorTest {
Iterator<Integer> expectedIterator = treeSet.iterator();
List<BaseFilterOperator> childOperators = new ArrayList<>();
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds1));
- childOperators.add(FilterOperatorTestUtils.makeFilterOperator(docIds2));
+ childOperators.add(new TestFilterOperator(docIds1));
+ childOperators.add(new TestFilterOperator(docIds2));
OrFilterOperator childOrOperator = new OrFilterOperator(childOperators);
List<BaseFilterOperator> operators = new ArrayList<>();
operators.add(childOrOperator);
- operators.add(FilterOperatorTestUtils.makeFilterOperator(docIds3));
+ operators.add(new TestFilterOperator(docIds3));
OrFilterOperator orOperator = new OrFilterOperator(operators);
BlockDocIdIterator iterator = orOperator.nextBlock().getBlockDocIdSet().iterator();
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/TestFilterOperator.java b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/TestFilterOperator.java
new file mode 100644
index 0000000..09e037e
--- /dev/null
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/operator/filter/TestFilterOperator.java
@@ -0,0 +1,78 @@
+/**
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.pinot.core.operator.filter;
+
+import com.linkedin.pinot.core.common.BlockDocIdIterator;
+import com.linkedin.pinot.core.operator.blocks.FilterBlock;
+import com.linkedin.pinot.core.operator.dociditerators.ArrayBasedDocIdIterator;
+import com.linkedin.pinot.core.operator.docidsets.FilterBlockDocIdSet;
+
+
+public class TestFilterOperator extends BaseFilterOperator {
+ private int[] _docIds;
+
+ public TestFilterOperator(int[] docIds) {
+ _docIds = docIds;
+ }
+
+ @Override
+ protected FilterBlock getNextBlock() {
+ return new FilterBlock(new FilterBlockDocIdSet() {
+ private int _minDocId = _docIds[0];
+ private int _maxDocId = _docIds[_docIds.length - 1];
+
+ @Override
+ public int getMinDocId() {
+ return _minDocId;
+ }
+
+ @Override
+ public int getMaxDocId() {
+ return _maxDocId;
+ }
+
+ @Override
+ public void setStartDocId(int startDocId) {
+ _minDocId = Math.max(_minDocId, startDocId);
+ }
+
+ @Override
+ public void setEndDocId(int endDocId) {
+ _maxDocId = Math.min(_maxDocId, endDocId);
+ }
+
+ @Override
+ public long getNumEntriesScannedInFilter() {
+ return 0;
+ }
+
+ @Override
+ public BlockDocIdIterator iterator() {
+ return new ArrayBasedDocIdIterator(_docIds, _docIds.length);
+ }
+
+ @Override
+ public <T> T getRaw() {
+ throw new UnsupportedOperationException();
+ }
+ });
+ }
+
+ @Override
+ public String getOperatorName() {
+ return "TestFilterOperator";
+ }
+}
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/core/predicate/RangeOfflineDictionaryPredicateEvaluatorTest.java b/pinot-core/src/test/java/com/linkedin/pinot/core/predicate/RangeOfflineDictionaryPredicateEvaluatorTest.java
index 10516e1..14d1815 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/core/predicate/RangeOfflineDictionaryPredicateEvaluatorTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/core/predicate/RangeOfflineDictionaryPredicateEvaluatorTest.java
@@ -32,13 +32,14 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
public void testRanges() {
int rangeStart, rangeEnd;
{
- // (2,5)
+ // [2, 5]
rangeStart = 2;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, true);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertTrue(evaluator.applySV(rangeStart));
Assert.assertTrue(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -52,13 +53,14 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
verifyDictId(dictIds, rangeStart, rangeEnd);
}
{
- // [2,5)
+ // (2, 5]
rangeStart = 2;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, false, rangeEnd, true);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertFalse(evaluator.applySV(rangeStart));
Assert.assertTrue(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -72,13 +74,14 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
verifyDictId(dictIds, rangeStart + 1, rangeEnd);
}
{
- // (2,5]
+ // [2, 5)
rangeStart = 2;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, false);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertTrue(evaluator.applySV(rangeStart));
Assert.assertFalse(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -92,13 +95,14 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
verifyDictId(dictIds, rangeStart, rangeEnd - 1);
}
{
- // [2,5]
+ // (2, 5)
rangeStart = 2;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, false, rangeEnd, false);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertFalse(evaluator.applySV(rangeStart));
Assert.assertFalse(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -124,13 +128,14 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
public void testBoundaries() {
int rangeStart, rangeEnd;
{
- // (0,5]
+ // [0, 5)
rangeStart = 0;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, false);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertTrue(evaluator.applySV(rangeStart));
Assert.assertFalse(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -143,26 +148,28 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
verifyDictId(dictIds, rangeStart, rangeEnd - 1);
}
{
- // (0,5)
+ // [0, 5]
rangeStart = 0;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, true);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertTrue(evaluator.applySV(rangeStart));
Assert.assertTrue(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
Assert.assertFalse(evaluator.applySV(rangeEnd + 1));
}
{
- // (6, DICT_LEN-1)
+ // [6, DICT_LEN-1]
rangeStart = 6;
rangeEnd = DICT_LEN - 1;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, true);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertTrue(evaluator.applySV(rangeStart));
Assert.assertTrue(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
@@ -175,30 +182,47 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
verifyDictId(dictIds, rangeStart, rangeEnd);
}
{
- // [6, DICT_LEN-1)
+ // (6, DICT_LEN-1]
rangeStart = 6;
rangeEnd = DICT_LEN - 1;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, false, rangeEnd, true);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertFalse(evaluator.applySV(rangeStart));
Assert.assertTrue(evaluator.applySV(rangeEnd));
Assert.assertTrue(evaluator.applySV(rangeStart + 1));
Assert.assertFalse(evaluator.applySV(rangeStart - 1));
}
+ {
+ // [0, DICT_LEN-1]
+ rangeStart = 0;
+ rangeEnd = DICT_LEN - 1;
+ ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
+ RangePredicate predicate = createPredicate(rangeStart, true, rangeEnd, true);
+ PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
+ Assert.assertFalse(evaluator.isAlwaysFalse());
+ Assert.assertTrue(evaluator.isAlwaysTrue());
+ Assert.assertTrue(evaluator.applySV(rangeStart));
+ Assert.assertTrue(evaluator.applySV(rangeEnd));
+ Assert.assertTrue(evaluator.applySV(rangeStart + 1));
+ Assert.assertFalse(evaluator.applySV(rangeStart - 1));
+ }
}
@Test
public void testZeroRange() {
int rangeStart, rangeEnd;
{
+ // (4, 5)
rangeStart = 4;
rangeEnd = 5;
ImmutableDictionaryReader reader = createReader(rangeStart, rangeEnd);
RangePredicate predicate = createPredicate(rangeStart, false, rangeEnd, false);
PredicateEvaluator evaluator = RangePredicateEvaluatorFactory.newDictionaryBasedEvaluator(predicate, reader);
Assert.assertTrue(evaluator.isAlwaysFalse());
+ Assert.assertFalse(evaluator.isAlwaysTrue());
Assert.assertFalse(evaluator.applySV(rangeStart));
Assert.assertFalse(evaluator.applySV(rangeEnd));
Assert.assertFalse(evaluator.applySV(rangeStart + 1));
@@ -225,11 +249,11 @@ public class RangeOfflineDictionaryPredicateEvaluatorTest {
when(predicate.includeLowerBoundary()).thenReturn(inclLower);
when(predicate.includeUpperBoundary()).thenReturn(inclUpper);
String lowerStr = "lower";
- if (lower == 0) {
+ if (lower == 0 && inclLower) {
lowerStr = "*";
}
String upperStr = "upper";
- if (upper == DICT_LEN - 1) {
+ if (upper == DICT_LEN - 1 && inclUpper) {
upperStr = "*";
}
when(predicate.getLowerBoundary()).thenReturn(lowerStr);
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java b/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java
deleted file mode 100644
index 58fb7db..0000000
--- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.linkedin.pinot.operator.filter;
-
-import com.linkedin.pinot.core.common.BlockDocIdIterator;
-import com.linkedin.pinot.core.operator.blocks.FilterBlock;
-import com.linkedin.pinot.core.operator.dociditerators.ArrayBasedDocIdIterator;
-import com.linkedin.pinot.core.operator.docidsets.FilterBlockDocIdSet;
-import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
-
-
-public class FilterOperatorTestUtils {
- private FilterOperatorTestUtils() {
- }
-
- public static BaseFilterOperator makeFilterOperator(int[] docIds) {
- return new BaseFilterOperator() {
- @Override
- public boolean isResultEmpty() {
- return docIds.length == 0;
- }
-
- @Override
- protected FilterBlock getNextBlock() {
- return new FilterBlock(new FilterBlockDocIdSet() {
- private int _minDocId = docIds[0];
- private int _maxDocId = docIds[docIds.length - 1];
-
- @Override
- public int getMinDocId() {
- return _minDocId;
- }
-
- @Override
- public int getMaxDocId() {
- return _maxDocId;
- }
-
- @Override
- public void setStartDocId(int startDocId) {
- _minDocId = Math.max(_minDocId, startDocId);
- }
-
- @Override
- public void setEndDocId(int endDocId) {
- _maxDocId = Math.min(_maxDocId, endDocId);
- }
-
- @Override
- public long getNumEntriesScannedInFilter() {
- return 0;
- }
-
- @Override
- public BlockDocIdIterator iterator() {
- return new ArrayBasedDocIdIterator(docIds, docIds.length);
- }
-
- @Override
- public <T> T getRaw() {
- return null;
- }
- });
- }
-
- @Override
- public String getOperatorName() {
- return null;
- }
- };
- }
-}
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentAggregationMultiValueQueriesTest.java b/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentAggregationMultiValueQueriesTest.java
index 1e866a8..152b1ed 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentAggregationMultiValueQueriesTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentAggregationMultiValueQueriesTest.java
@@ -52,7 +52,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
// Test query with filter.
aggregationOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationOperator.nextBlock();
- QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(), 15620L, 282430L,
+ QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(), 15620L, 263414L,
62480L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationResult(resultsBlock.getAggregationResult(), 15620L, 17287754700747L,
999943053, 1182655, 11017594448983L, 15620L);
@@ -73,7 +73,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
// Test query with filter.
aggregationOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationOperator.nextBlock();
- QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(), 15620L, 282430L,
+ QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(), 15620L, 263414L,
31240L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationResult(resultsBlock.getAggregationResult(), 15620L, 28567975886777L,
2147483647, 203, 28663153397978L, 15620L);
@@ -95,7 +95,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
aggregationGroupByOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationGroupByOperator.nextBlock();
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationGroupByOperator.getExecutionStatistics(), 15620L,
- 282430L, 78100L, 100000L);
+ 263414L, 78100L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationGroupByResult(resultsBlock.getAggregationGroupByResult(), "203", 1L,
185436225L, 987549258, 674022574, 674022574L, 1L);
}
@@ -116,7 +116,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
aggregationGroupByOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationGroupByOperator.nextBlock();
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationGroupByOperator.getExecutionStatistics(), 15620L,
- 282430L, 109340L, 100000L);
+ 263414L, 109340L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationGroupByResult(resultsBlock.getAggregationGroupByResult(),
"L\t306633\t2147483647", 1L, 131154783L, 952002176, 674022574, 674022574L, 1L);
}
@@ -137,7 +137,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
aggregationGroupByOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationGroupByOperator.nextBlock();
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationGroupByOperator.getExecutionStatistics(), 15620L,
- 282430L, 109340L, 100000L);
+ 263414L, 109340L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationGroupByResult(resultsBlock.getAggregationGroupByResult(),
"903461357\tL\t2147483647\t388", 2L, 1806922714L, 652024397, 674022574, 870535054L, 2L);
}
@@ -159,7 +159,7 @@ public class InnerSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
aggregationGroupByOperator = getOperatorForQueryWithFilter(query);
resultsBlock = aggregationGroupByOperator.nextBlock();
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationGroupByOperator.getExecutionStatistics(), 15620L,
- 282430L, 109340L, 100000L);
+ 263414L, 109340L, 100000L);
QueriesTestUtils.testInnerSegmentAggregationGroupByResult(resultsBlock.getAggregationGroupByResult(),
"949960647\t238753654\t2147483647\t2147483647\t674022574\t674022574\t674022574", 2L, 1899921294L, 238753654,
674022574, 1348045148L, 2L);
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentSelectionMultiValueQueriesTest.java b/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentSelectionMultiValueQueriesTest.java
index a53f0ee..6048a41 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentSelectionMultiValueQueriesTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/queries/InnerSegmentSelectionMultiValueQueriesTest.java
@@ -100,7 +100,7 @@ public class InnerSegmentSelectionMultiValueQueriesTest extends BaseMultiValueQu
resultsBlock = selectionOnlyOperator.nextBlock();
executionStatistics = selectionOnlyOperator.getExecutionStatistics();
Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
- Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 230501L);
+ Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 77L);
Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 100L);
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
selectionDataSchema = resultsBlock.getSelectionDataSchema();
@@ -147,7 +147,7 @@ public class InnerSegmentSelectionMultiValueQueriesTest extends BaseMultiValueQu
resultsBlock = selectionOnlyOperator.nextBlock();
executionStatistics = selectionOnlyOperator.getExecutionStatistics();
Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
- Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 230501L);
+ Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 77L);
Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 30L);
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
selectionDataSchema = resultsBlock.getSelectionDataSchema();
@@ -194,7 +194,7 @@ public class InnerSegmentSelectionMultiValueQueriesTest extends BaseMultiValueQu
resultsBlock = selectionOrderByOperator.nextBlock();
executionStatistics = selectionOrderByOperator.getExecutionStatistics();
Assert.assertEquals(executionStatistics.getNumDocsScanned(), 15620L);
- Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 282430L);
+ Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 263414L);
Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 62480L);
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
selectionDataSchema = resultsBlock.getSelectionDataSchema();
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/queries/InterSegmentAggregationMultiValueQueriesTest.java b/pinot-core/src/test/java/com/linkedin/pinot/queries/InterSegmentAggregationMultiValueQueriesTest.java
index 072ff43..ae3fcae 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/queries/InterSegmentAggregationMultiValueQueriesTest.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/queries/InterSegmentAggregationMultiValueQueriesTest.java
@@ -35,7 +35,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"426752"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"62480"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -56,7 +56,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -77,7 +77,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"1001.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"1009.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -98,7 +98,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"484324601810280.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"114652613591912.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -119,7 +119,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"1134908803.73210"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"1835029026.75916"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -140,7 +140,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147482646.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147482638.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -161,7 +161,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"18499"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"1186"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -182,7 +182,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"20039"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"1296"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -203,7 +203,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -224,7 +224,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -245,7 +245,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -266,7 +266,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647.00000"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -287,7 +287,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -308,7 +308,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -329,7 +329,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
@@ -350,7 +350,7 @@ public class InterSegmentAggregationMultiValueQueriesTest extends BaseMultiValue
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQueryWithFilter(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1129720L, 62480L, 400000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 62480L, 1053656L, 62480L, 400000L,
new String[]{"2147483647"});
brokerResponse = getBrokerResponseForQuery(query + SV_GROUP_BY);
diff --git a/pinot-integration-tests/src/test/java/com/linkedin/pinot/integration/tests/MetadataAndDictionaryAggregationPlanClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/com/linkedin/pinot/integration/tests/MetadataAndDictionaryAggregationPlanClusterIntegrationTest.java
index 76d5303..c5cfee0 100644
--- a/pinot-integration-tests/src/test/java/com/linkedin/pinot/integration/tests/MetadataAndDictionaryAggregationPlanClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/com/linkedin/pinot/integration/tests/MetadataAndDictionaryAggregationPlanClusterIntegrationTest.java
@@ -15,14 +15,10 @@
*/
package com.linkedin.pinot.integration.tests;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.linkedin.pinot.common.data.Schema;
-import com.linkedin.pinot.common.utils.CommonConstants;
-import com.linkedin.pinot.common.utils.ServiceStatus;
import com.linkedin.pinot.util.TestUtils;
import java.io.File;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -50,14 +46,11 @@ import org.testng.annotations.Test;
* </li>
* </ul>
*/
+// TODO: remove this integration test and add unit test for metadata and dictionary based aggregation operator
public class MetadataAndDictionaryAggregationPlanClusterIntegrationTest extends BaseClusterIntegrationTest {
-
private static final int NUM_BROKERS = 1;
private static final int NUM_SERVERS = 1;
- private final List<ServiceStatus.ServiceStatusCallback> _serviceStatusCallbacks =
- new ArrayList<>(getNumBrokers() + getNumServers());
-
protected int getNumBrokers() {
return NUM_BROKERS;
}
@@ -97,22 +90,6 @@ public class MetadataAndDictionaryAggregationPlanClusterIntegrationTest extends
startBrokers(getNumBrokers());
startServers(getNumServers());
- // Set up service status callbacks
- List<String> instances = _helixAdmin.getInstancesInCluster(_clusterName);
- for (String instance : instances) {
- if (instance.startsWith(CommonConstants.Helix.PREFIX_OF_BROKER_INSTANCE)) {
- _serviceStatusCallbacks.add(
- new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(_helixManager, _clusterName, instance,
- Collections.singletonList(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)));
- }
- if (instance.startsWith(CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE)) {
- _serviceStatusCallbacks.add(new ServiceStatus.MultipleCallbackServiceStatusCallback(ImmutableList.of(
- new ServiceStatus.IdealStateAndCurrentStateMatchServiceStatusCallback(_helixManager, _clusterName,
- instance),
- new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(_helixManager, _clusterName,
- instance))));
- }
- }
// Create the tables
addOfflineTable(DEFAULT_TABLE_NAME);
addOfflineTable(STAR_TREE_TABLE_NAME);
@@ -423,11 +400,10 @@ public class MetadataAndDictionaryAggregationPlanClusterIntegrationTest extends
Assert.assertEquals(response.getLong("totalDocs"), response.getLong("numDocsScanned"));
// filter in query: not answered by DictionaryBasedAggregationOperator
- pqlQuery = "SELECT MAX(ArrTime) FROM " + DEFAULT_TABLE_NAME + " where DaysSinceEpoch > 0";
+ pqlQuery = "SELECT MAX(ArrTime) FROM " + DEFAULT_TABLE_NAME + " where DaysSinceEpoch > 16100";
response = postQuery(pqlQuery);
Assert.assertEquals(response.getLong("numEntriesScannedPostFilter") > 0, true);
Assert.assertEquals(response.getLong("numEntriesScannedInFilter") > 0, true);
- Assert.assertEquals(response.getLong("totalDocs"), response.getLong("numDocsScanned"));
}
@@ -476,11 +452,10 @@ public class MetadataAndDictionaryAggregationPlanClusterIntegrationTest extends
Assert.assertEquals(response.getLong("totalDocs"), response.getLong("numDocsScanned"));
// filter present in query: not answered by MetadataBasedAggregationOperator
- pqlQuery = "SELECT COUNT(*) FROM " + DEFAULT_TABLE_NAME + " WHERE DaysSinceEpoch > 0";
+ pqlQuery = "SELECT COUNT(*) FROM " + DEFAULT_TABLE_NAME + " WHERE DaysSinceEpoch > 16100";
response = postQuery(pqlQuery);
Assert.assertEquals(response.getLong("numEntriesScannedPostFilter"), 0);
Assert.assertEquals(response.getLong("numEntriesScannedInFilter") > 0, true);
- Assert.assertEquals(response.getLong("totalDocs"), response.getLong("numDocsScanned"));
// mixed aggregation functions in query: not answered by MetadataBasedAggregationOperator
pqlQuery = "SELECT COUNT(*),MAX(ArrTime) FROM " + DEFAULT_TABLE_NAME;
diff --git a/pinot-perf/src/main/java/com/linkedin/pinot/perf/RawIndexBenchmark.java b/pinot-perf/src/main/java/com/linkedin/pinot/perf/RawIndexBenchmark.java
index 1f6a5c9..416db31 100644
--- a/pinot-perf/src/main/java/com/linkedin/pinot/perf/RawIndexBenchmark.java
+++ b/pinot-perf/src/main/java/com/linkedin/pinot/perf/RawIndexBenchmark.java
@@ -30,10 +30,10 @@ import com.linkedin.pinot.core.operator.ProjectionOperator;
import com.linkedin.pinot.core.operator.blocks.ProjectionBlock;
import com.linkedin.pinot.core.operator.docvalsets.ProjectionBlockValSet;
import com.linkedin.pinot.core.operator.filter.BaseFilterOperator;
+import com.linkedin.pinot.core.operator.filter.TestFilterOperator;
import com.linkedin.pinot.core.plan.DocIdSetPlanNode;
import com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl;
import com.linkedin.pinot.core.segment.creator.impl.V1Constants;
-import com.linkedin.pinot.operator.filter.FilterOperatorTestUtils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@ -73,7 +73,8 @@ public class RawIndexBenchmark {
@Option(name = "-rawIndexColumn", required = false, usage = "Name of column with raw index (no-dictionary")
private String _rawIndexColumn = DEFAULT_RAW_INDEX_COLUMN;
- @Option(name = "-dataFile", required = false, forbids = {"-segmentDir"}, usage = "File containing input data (one string per line)")
+ @Option(name = "-dataFile", required = false, forbids = {
+ "-segmentDir"}, usage = "File containing input data (one string per line)")
private String _dataFile = null;
@Option(name = "-loadMode", required = false, usage = "Load mode for data (mmap|heap")
@@ -209,7 +210,7 @@ public class RawIndexBenchmark {
* @return Time take in millis for the lookups
*/
private long profileLookups(IndexSegment segment, String column, int[] docIds) {
- BaseFilterOperator filterOperator = FilterOperatorTestUtils.makeFilterOperator(docIds);
+ BaseFilterOperator filterOperator = new TestFilterOperator(docIds);
DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL);
ProjectionOperator projectionOperator = new ProjectionOperator(buildDataSourceMap(segment), docIdSetOperator);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org