You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/03/05 00:43:26 UTC
[incubator-pinot] branch fixing_transform_docIds_thread_local_issue
created (now 82833f2)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch fixing_transform_docIds_thread_local_issue
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 82833f2 Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator
This branch includes the following new commits:
new 82833f2 Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Bugfixing the issue for ThreadLocal
DocIdSet in ExpressionFilterOperator
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch fixing_transform_docIds_thread_local_issue
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 82833f21a0c7b88c0083df1d5f90683e430cc47b
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Mar 4 16:42:57 2020 -0800
Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator
---
.../org/apache/pinot/core/operator/DocIdSetOperator.java | 8 +++++++-
.../pinot/core/operator/filter/ExpressionFilterOperator.java | 2 +-
.../integration/tests/BaseClusterIntegrationTestSet.java | 12 ++++++++++++
3 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
index 8528ef9..133db262 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
@@ -49,11 +49,17 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
private FilterBlockDocIdSet _filterBlockDocIdSet;
private BlockDocIdIterator _blockDocIdIterator;
private int _currentDocId = 0;
+ private boolean _threadLocal = true;
public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet) {
+ this(filterOperator, maxSizeOfDocIdSet, true);
+ }
+
+ public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet, boolean threadLocal) {
Preconditions.checkArgument(maxSizeOfDocIdSet > 0 && maxSizeOfDocIdSet <= DocIdSetPlanNode.MAX_DOC_PER_CALL);
_filterOperator = filterOperator;
_maxSizeOfDocIdSet = maxSizeOfDocIdSet;
+ _threadLocal = threadLocal;
}
@Override
@@ -69,7 +75,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
}
int pos = 0;
- int[] docIds = THREAD_LOCAL_DOC_IDS.get();
+ int[] docIds = _threadLocal? THREAD_LOCAL_DOC_IDS.get(): new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
for (int i = 0; i < _maxSizeOfDocIdSet; i++) {
_currentDocId = _blockDocIdIterator.next();
if (_currentDocId == Constants.EOF) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
index 1f7a6d3..718d879 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
@@ -290,7 +290,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) {
BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer);
- DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL);
+ DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false);
ProjectionOperator projectionOperator =
new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator);
TransformOperator operator =
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
index 52b6fcb..52df943 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
@@ -170,6 +170,18 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati
String query;
List<String> h2queries;
query =
+ "SELECT COUNT(*) FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT COUNT(*) FROM mytable WHERE ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
"SELECT count(*) FROM mytable WHERE AirlineID > 20355 AND OriginState BETWEEN 'PA' AND 'DE' AND DepTime <> 2202 LIMIT 21";
testSqlQuery(query, Collections.singletonList(query));
query =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org