You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/03/07 09:14:57 UTC
[incubator-pinot] 01/06: Bugfixing the issue for ThreadLocal
DocIdSet in ExpressionFilterOperator (#5114)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 6f00be37ff467ddb163a6836c6ab6fb478072a60
Author: Jialiang Li <jl...@linkedin.com>
AuthorDate: Thu Mar 5 13:24:19 2020 -0800
Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator (#5114)
* Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator
* Update bitmap range start/end id
* Address comments
---
.../org/apache/pinot/core/operator/DocIdSetOperator.java | 8 +++++++-
.../pinot/core/operator/filter/ExpressionFilterOperator.java | 7 ++++---
.../integration/tests/BaseClusterIntegrationTestSet.java | 12 ++++++++++++
3 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
index 8528ef9..5db7db4 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
@@ -49,11 +49,17 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
private FilterBlockDocIdSet _filterBlockDocIdSet;
private BlockDocIdIterator _blockDocIdIterator;
private int _currentDocId = 0;
+ private boolean _threadLocal = true;
public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet) {
+ this(filterOperator, maxSizeOfDocIdSet, true);
+ }
+
+ public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet, boolean threadLocal) {
Preconditions.checkArgument(maxSizeOfDocIdSet > 0 && maxSizeOfDocIdSet <= DocIdSetPlanNode.MAX_DOC_PER_CALL);
_filterOperator = filterOperator;
_maxSizeOfDocIdSet = maxSizeOfDocIdSet;
+ _threadLocal = threadLocal;
}
@Override
@@ -69,7 +75,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
}
int pos = 0;
- int[] docIds = THREAD_LOCAL_DOC_IDS.get();
+ int[] docIds = _threadLocal? THREAD_LOCAL_DOC_IDS.get(): new int[_maxSizeOfDocIdSet];
for (int i = 0; i < _maxSizeOfDocIdSet; i++) {
_currentDocId = _blockDocIdIterator.next();
if (_currentDocId == Constants.EOF) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
index 1f7a6d3..9078403 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
@@ -151,7 +151,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
throw new UnsupportedOperationException("Filter on expressions that return multi-values is not yet supported");
}
_blockDocIdIterator.setStartDocId(0);
- _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs - 1);
+ _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs);
}
@Override
@@ -223,8 +223,9 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
_currentBlockEndDocId = _currentBlockStartDocId + DocIdSetPlanNode.MAX_DOC_PER_CALL;
_currentBlockEndDocId = Math.min(_currentBlockEndDocId, _endDocId);
MutableRoaringBitmap bitmapRange = new MutableRoaringBitmap();
- bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId + 1);
+ bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId);
MutableRoaringBitmap matchedBitmap = evaluate(bitmapRange);
+
_intIterator = matchedBitmap.getIntIterator();
_numDocsScanned += (_currentBlockEndDocId - _currentBlockStartDocId);
}
@@ -290,7 +291,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) {
BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer);
- DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL);
+ DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false);
ProjectionOperator projectionOperator =
new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator);
TransformOperator operator =
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
index 52b6fcb..52df943 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
@@ -170,6 +170,18 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati
String query;
List<String> h2queries;
query =
+ "SELECT COUNT(*) FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT COUNT(*) FROM mytable WHERE ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
"SELECT count(*) FROM mytable WHERE AirlineID > 20355 AND OriginState BETWEEN 'PA' AND 'DE' AND DepTime <> 2202 LIMIT 21";
testSqlQuery(query, Collections.singletonList(query));
query =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org