You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/03/07 09:14:56 UTC
[incubator-pinot] branch release-0.3.0 updated (039e189 -> 4a878f2)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 039e189 [maven-release-plugin] prepare for next development iteration
new 6f00be3 Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator (#5114)
new e081134 Fix the ExpressionFilterOperator (#5122)
new e66df48 Fix default value for dictionary byte column (#5123)
new 91c66be Defaulttenant for hybrid quickstart and example queriesDefault tenant and example queries for hybrid quickstart
new 23b5c4f Always copy test data out from jar file (#5119)
new 4a878f2 [maven-release-plugin] prepare release release-0.3.0-rc1
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis_quickstart.sh | 49 ++++++++++++-
.../airlineStats_realtime_table_config.json | 5 +-
kubernetes/helm/pinot-realtime-quickstart.yml | 10 +--
pinot-broker/pom.xml | 2 +-
pinot-clients/pinot-java-client/pom.xml | 2 +-
pinot-clients/pom.xml | 2 +-
pinot-common/pom.xml | 2 +-
pinot-controller/pom.xml | 2 +-
pinot-core/pom.xml | 2 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 6 +-
.../pinot/core/minion/RawIndexConverter.java | 6 +-
.../pinot/core/operator/DocIdSetOperator.java | 8 ++-
.../operator/filter/ExpressionFilterOperator.java | 52 +++++++-------
pinot-distribution/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
.../tests/BaseClusterIntegrationTestSet.java | 12 ++++
pinot-minion/pom.xml | 2 +-
pinot-perf/pom.xml | 2 +-
.../pinot-batch-ingestion-common/pom.xml | 2 +-
.../pinot-batch-ingestion-hadoop/pom.xml | 2 +-
.../pinot-batch-ingestion-spark/pom.xml | 2 +-
.../pinot-batch-ingestion-standalone/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/pom.xml | 2 +-
.../v0_deprecated/pinot-hadoop/pom.xml | 2 +-
.../v0_deprecated/pinot-ingestion-common/pom.xml | 2 +-
.../v0_deprecated/pinot-spark/pom.xml | 2 +-
.../pinot-batch-ingestion/v0_deprecated/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pom.xml | 2 +-
.../pinot-input-format/pinot-avro/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-csv/pom.xml | 2 +-
.../pinot-input-format/pinot-json/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-orc/pom.xml | 2 +-
.../pinot-input-format/pinot-parquet/pom.xml | 2 +-
.../pinot-input-format/pinot-thrift/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pom.xml | 2 +-
.../pinot-stream-ingestion/pinot-kafka-0.9/pom.xml | 2 +-
.../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 2 +-
.../pinot-kafka-base/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pom.xml | 2 +-
pinot-plugins/pom.xml | 2 +-
pinot-server/pom.xml | 2 +-
pinot-spi/pom.xml | 2 +-
pinot-tools/pom.xml | 2 +-
.../org/apache/pinot/tools/HybridQuickstart.java | 46 ++++++++++--
.../tools/admin/command/QuickstartRunner.java | 10 ++-
.../org/apache/pinot/tools/utils/JarUtils.java | 83 ++++++++++++++++++++++
.../airlineStats_offline_table_config.json | 5 +-
.../airlineStats_realtime_table_config.json | 5 +-
.../docker/airlineStats_realtime_table_config.json | 5 +-
.../airlineStats_realtime_table_config.json | 5 +-
.../airlineStats_realtime_table_config.json | 5 +-
pom.xml | 4 +-
55 files changed, 277 insertions(+), 115 deletions(-)
create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 02/06: Fix the ExpressionFilterOperator (#5122)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e08113449e02ccc4dde76be28f84ab201fa694a6
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Fri Mar 6 17:41:16 2020 -0800
Fix the ExpressionFilterOperator (#5122)
Fix the bug of using inclusive endDocId as exclusive, which causes
matching documents not being selected
TODO: Change filter to always use exclusive endDocId
---
.../operator/filter/ExpressionFilterOperator.java | 47 ++++++++++------------
1 file changed, 21 insertions(+), 26 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
index 9078403..e1d12ef 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
@@ -161,7 +161,8 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
@Override
public int getMaxDocId() {
- return _blockDocIdIterator._endDocId;
+ // NOTE: Return value should be inclusive
+ return _blockDocIdIterator._endDocId - 1;
}
@Override
@@ -171,7 +172,8 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
@Override
public void setEndDocId(int endDocId) {
- _blockDocIdIterator.setEndDocId(endDocId);
+ // NOTE: The passed in argument is inclusive
+ _blockDocIdIterator.setEndDocId(endDocId + 1);
}
@Override
@@ -193,6 +195,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
private ExpressionFilterOperator _expressionFilterOperator;
int _numDocsScanned = 0;
int _startDocId;
+ // Exclusive
int _endDocId;
//used only in next() and advance methods
int _currentBlockStartDocId = -1;
@@ -217,11 +220,17 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
if (_currentDocId == Constants.EOF) {
return Constants.EOF;
}
- while (_currentDocId < _endDocId) {
- if (_intIterator == null) {
+ while (true) {
+ if (_intIterator != null && _intIterator.hasNext()) {
+ _currentDocId = _intIterator.next();
+ return _currentDocId;
+ } else {
+ if (_currentBlockEndDocId == _endDocId) {
+ _currentDocId = Constants.EOF;
+ return _currentDocId;
+ }
_currentBlockStartDocId = _currentBlockEndDocId;
- _currentBlockEndDocId = _currentBlockStartDocId + DocIdSetPlanNode.MAX_DOC_PER_CALL;
- _currentBlockEndDocId = Math.min(_currentBlockEndDocId, _endDocId);
+ _currentBlockEndDocId = Math.min(_currentBlockStartDocId + DocIdSetPlanNode.MAX_DOC_PER_CALL, _endDocId);
MutableRoaringBitmap bitmapRange = new MutableRoaringBitmap();
bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId);
MutableRoaringBitmap matchedBitmap = evaluate(bitmapRange);
@@ -229,32 +238,19 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
_intIterator = matchedBitmap.getIntIterator();
_numDocsScanned += (_currentBlockEndDocId - _currentBlockStartDocId);
}
- if (_intIterator.hasNext()) {
- return (_currentDocId = _intIterator.next());
- } else {
- _currentDocId = _currentBlockEndDocId;
- _intIterator = null;
- }
}
- _currentDocId = Constants.EOF;
- return _currentDocId;
}
@Override
public int advance(int targetDocId) {
- if (_currentDocId == Constants.EOF) {
+ if (_currentDocId == Constants.EOF || _currentDocId >= targetDocId) {
return _currentDocId;
}
- if (targetDocId < _startDocId) {
- targetDocId = _startDocId;
- } else if (targetDocId > _endDocId) {
+ if (targetDocId >= _endDocId) {
_currentDocId = Constants.EOF;
return _currentDocId;
}
if (targetDocId >= _currentBlockStartDocId && targetDocId < _currentBlockEndDocId) {
- if (_currentDocId == targetDocId) {
- return _currentDocId;
- }
while (_intIterator.hasNext()) {
_currentDocId = _intIterator.next();
if (_currentDocId >= targetDocId) {
@@ -271,7 +267,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
@Override
public int currentDocId() {
- return 0;
+ return _currentDocId;
}
@Override
@@ -284,14 +280,13 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
@Override
public MutableRoaringBitmap applyAnd(MutableRoaringBitmap answer) {
- MutableRoaringBitmap bitmap = evaluate(answer);
- answer.and(bitmap);
- return answer;
+ return evaluate(answer);
}
private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) {
BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer);
- DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false);
+ DocIdSetOperator docIdSetOperator =
+ new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false);
ProjectionOperator projectionOperator =
new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator);
TransformOperator operator =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 06/06: [maven-release-plugin] prepare release
release-0.3.0-rc1
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 4a878f21aac289c84f289e47958de64850b7bb81
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Sat Mar 7 01:14:42 2020 -0800
[maven-release-plugin] prepare release release-0.3.0-rc1
---
pinot-broker/pom.xml | 2 +-
pinot-clients/pinot-java-client/pom.xml | 2 +-
pinot-clients/pom.xml | 2 +-
pinot-common/pom.xml | 2 +-
pinot-controller/pom.xml | 2 +-
pinot-core/pom.xml | 2 +-
pinot-distribution/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
pinot-minion/pom.xml | 2 +-
pinot-perf/pom.xml | 2 +-
.../pinot-batch-ingestion/pinot-batch-ingestion-common/pom.xml | 2 +-
.../pinot-batch-ingestion/pinot-batch-ingestion-hadoop/pom.xml | 2 +-
.../pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml | 2 +-
.../pinot-batch-ingestion/pinot-batch-ingestion-standalone/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/pom.xml | 2 +-
.../pinot-batch-ingestion/v0_deprecated/pinot-hadoop/pom.xml | 2 +-
.../v0_deprecated/pinot-ingestion-common/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/v0_deprecated/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-avro/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-csv/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-json/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-orc/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-parquet/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-thrift/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pom.xml | 2 +-
pinot-plugins/pom.xml | 2 +-
pinot-server/pom.xml | 2 +-
pinot-spi/pom.xml | 2 +-
pinot-tools/pom.xml | 2 +-
pom.xml | 4 ++--
39 files changed, 40 insertions(+), 40 deletions(-)
diff --git a/pinot-broker/pom.xml b/pinot-broker/pom.xml
index ea85d4e..2236869 100644
--- a/pinot-broker/pom.xml
+++ b/pinot-broker/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-broker</artifactId>
<name>Pinot Broker</name>
diff --git a/pinot-clients/pinot-java-client/pom.xml b/pinot-clients/pinot-java-client/pom.xml
index ef96fb7..e7c105c 100644
--- a/pinot-clients/pinot-java-client/pom.xml
+++ b/pinot-clients/pinot-java-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-clients</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-java-client</artifactId>
<name>Pinot Java Client</name>
diff --git a/pinot-clients/pom.xml b/pinot-clients/pom.xml
index b7069a0..dfdbe67 100644
--- a/pinot-clients/pom.xml
+++ b/pinot-clients/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-clients</artifactId>
<packaging>pom</packaging>
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index 49cb36b..c61a6b6 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-common</artifactId>
<name>Pinot Common</name>
diff --git a/pinot-controller/pom.xml b/pinot-controller/pom.xml
index 12f5cfb..bbeb37e 100644
--- a/pinot-controller/pom.xml
+++ b/pinot-controller/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-controller</artifactId>
<name>Pinot Controller</name>
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index 7647822..81be396 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-core</artifactId>
<name>Pinot Core</name>
diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml
index 1f60f5e..14d40e2 100644
--- a/pinot-distribution/pom.xml
+++ b/pinot-distribution/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-distribution</artifactId>
<name>Pinot Distribution</name>
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index bab3005..f971e9f 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-integration-tests</artifactId>
<name>Pinot Integration Tests</name>
diff --git a/pinot-minion/pom.xml b/pinot-minion/pom.xml
index 067be6f..56c6753 100644
--- a/pinot-minion/pom.xml
+++ b/pinot-minion/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-minion</artifactId>
<name>Pinot Minion</name>
diff --git a/pinot-perf/pom.xml b/pinot-perf/pom.xml
index e23fbee..69a076e 100644
--- a/pinot-perf/pom.xml
+++ b/pinot-perf/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-perf</artifactId>
<name>Pinot Perf</name>
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/pom.xml
index fd740cb..ed70d21 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-batch-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/pom.xml
index 0299a70..3b6250b 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-batch-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml
index df6bcd5..94bbc8f 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-batch-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pom.xml
index ea58e07..2b7f7b7 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-batch-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-batch-ingestion/pom.xml b/pinot-plugins/pinot-batch-ingestion/pom.xml
index 67a9a8e..70a7a0e 100644
--- a/pinot-plugins/pinot-batch-ingestion/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-plugins</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-batch-ingestion</artifactId>
<packaging>pom</packaging>
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/pom.xml b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/pom.xml
index c43a833..91113f4 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>v0_deprecated</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-hadoop</artifactId>
<name>Pinot Hadoop</name>
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-ingestion-common/pom.xml b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-ingestion-common/pom.xml
index 0d2484b..69acab7 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-ingestion-common/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-ingestion-common/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>v0_deprecated</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<relativePath>..</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml
index c617f5a..e9af767 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>v0_deprecated</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-spark</artifactId>
<name>Pinot Spark</name>
diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pom.xml b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pom.xml
index da7c25d..cb1f0bc 100644
--- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-batch-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>v0_deprecated</artifactId>
diff --git a/pinot-plugins/pinot-file-system/pinot-adls/pom.xml b/pinot-plugins/pinot-file-system/pinot-adls/pom.xml
index c502fb5..a9a423c 100644
--- a/pinot-plugins/pinot-file-system/pinot-adls/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-adls/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-file-system</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-adls</artifactId>
<groupId>org.apache.pinot.plugins</groupId>
diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
index 7225b12..c77088d 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-file-system</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-gcs</artifactId>
diff --git a/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml b/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml
index 616ea17..043e265 100644
--- a/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-file-system</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-hdfs</artifactId>
<name>Pinot Hadoop Filesystem</name>
diff --git a/pinot-plugins/pinot-file-system/pom.xml b/pinot-plugins/pinot-file-system/pom.xml
index 830e0b6..a302e9c 100644
--- a/pinot-plugins/pinot-file-system/pom.xml
+++ b/pinot-plugins/pinot-file-system/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-plugins</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-file-system</artifactId>
diff --git a/pinot-plugins/pinot-input-format/pinot-avro/pom.xml b/pinot-plugins/pinot-input-format/pinot-avro/pom.xml
index 0b1190d..ce831f1 100644
--- a/pinot-plugins/pinot-input-format/pinot-avro/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-avro/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pinot-csv/pom.xml b/pinot-plugins/pinot-input-format/pinot-csv/pom.xml
index 5d96fd1..88e1be3 100644
--- a/pinot-plugins/pinot-input-format/pinot-csv/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-csv/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pinot-json/pom.xml b/pinot-plugins/pinot-input-format/pinot-json/pom.xml
index a0f8e44..bf149ea 100644
--- a/pinot-plugins/pinot-input-format/pinot-json/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-json/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pinot-orc/pom.xml b/pinot-plugins/pinot-input-format/pinot-orc/pom.xml
index 2bcb18b..9f0bbb3 100644
--- a/pinot-plugins/pinot-input-format/pinot-orc/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-orc/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml b/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml
index f039733..4db0c58 100644
--- a/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pinot-thrift/pom.xml b/pinot-plugins/pinot-input-format/pinot-thrift/pom.xml
index b582bbc..50d3d69 100644
--- a/pinot-plugins/pinot-input-format/pinot-thrift/pom.xml
+++ b/pinot-plugins/pinot-input-format/pinot-thrift/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-input-format</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pinot-plugins/pinot-input-format/pom.xml b/pinot-plugins/pinot-input-format/pom.xml
index 791fe4b..c9dc381 100644
--- a/pinot-plugins/pinot-input-format/pom.xml
+++ b/pinot-plugins/pinot-input-format/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-plugins</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-input-format</artifactId>
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml
index 0af51e5..77e7fc3 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-stream-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pinot-kafka-0.9</artifactId>
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml
index ac4a27e..9747377 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-stream-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pinot-kafka-2.0</artifactId>
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/pom.xml
index ba79fab..8b570e7 100644
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/pom.xml
+++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>pinot-stream-ingestion</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pinot-kafka-base</artifactId>
diff --git a/pinot-plugins/pinot-stream-ingestion/pom.xml b/pinot-plugins/pinot-stream-ingestion/pom.xml
index 5ea0577..5eb0a04 100644
--- a/pinot-plugins/pinot-stream-ingestion/pom.xml
+++ b/pinot-plugins/pinot-stream-ingestion/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot-plugins</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-stream-ingestion</artifactId>
<packaging>pom</packaging>
diff --git a/pinot-plugins/pom.xml b/pinot-plugins/pom.xml
index e159f71..5a82bd0 100644
--- a/pinot-plugins/pom.xml
+++ b/pinot-plugins/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-plugins</artifactId>
<packaging>pom</packaging>
diff --git a/pinot-server/pom.xml b/pinot-server/pom.xml
index bd12f45..604f223 100644
--- a/pinot-server/pom.xml
+++ b/pinot-server/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-server</artifactId>
<name>Pinot Server</name>
diff --git a/pinot-spi/pom.xml b/pinot-spi/pom.xml
index 9efdc07..eb3e93c 100644
--- a/pinot-spi/pom.xml
+++ b/pinot-spi/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-spi</artifactId>
<name>Pinot Service Provider Interface</name>
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index 8cd91ac..3fb8f1d 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>pinot</artifactId>
<groupId>org.apache.pinot</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
</parent>
<artifactId>pinot-tools</artifactId>
<name>Pinot Tools</name>
diff --git a/pom.xml b/pom.xml
index d27be42..5c05295 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.3.0</version>
<packaging>pom</packaging>
<name>Pinot</name>
<description>A realtime distributed OLAP datastore</description>
@@ -90,7 +90,7 @@
<scm>
<developerConnection>scm:git:git@github.com:apache/incubator-pinot.git</developerConnection>
- <tag>HEAD</tag>
+ <tag>release-0.3.0-rc1</tag>
</scm>
<!-- Apache project inception year for generating correct NOTICE file for jar bundle. -->
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 04/06: Defaulttenant for hybrid quickstart and
example queriesDefault tenant and example queries for hybrid quickstart
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 91c66be4c9737bc239c0e9d1f1a6f64f4fd5bb84
Author: Neha Pawar <np...@linkedin.com>
AuthorDate: Mon Mar 2 17:11:22 2020 -0800
Defaulttenant for hybrid quickstart and example queriesDefault tenant and example queries for hybrid quickstart
---
.../airlineStats_realtime_table_config.json | 5 +--
kubernetes/helm/pinot-realtime-quickstart.yml | 10 +----
.../org/apache/pinot/tools/HybridQuickstart.java | 43 +++++++++++++++++++---
.../airlineStats_offline_table_config.json | 5 +--
.../airlineStats_realtime_table_config.json | 5 +--
.../docker/airlineStats_realtime_table_config.json | 5 +--
.../airlineStats_realtime_table_config.json | 5 +--
.../airlineStats_realtime_table_config.json | 5 +--
8 files changed, 46 insertions(+), 37 deletions(-)
diff --git a/docker/images/pinot/table-configs/airlineStats_realtime_table_config.json b/docker/images/pinot/table-configs/airlineStats_realtime_table_config.json
index f8e80b1..c0d6890 100644
--- a/docker/images/pinot/table-configs/airlineStats_realtime_table_config.json
+++ b/docker/images/pinot/table-configs/airlineStats_realtime_table_config.json
@@ -12,10 +12,7 @@
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
diff --git a/kubernetes/helm/pinot-realtime-quickstart.yml b/kubernetes/helm/pinot-realtime-quickstart.yml
index 52e7ae3..f50c977 100644
--- a/kubernetes/helm/pinot-realtime-quickstart.yml
+++ b/kubernetes/helm/pinot-realtime-quickstart.yml
@@ -38,10 +38,7 @@ data:
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
@@ -78,10 +75,7 @@ data:
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 3d2dfb0..7951a3f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URL;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.config.TagNameUtils;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.spi.data.readers.FileFormat;
@@ -35,6 +36,7 @@ import org.apache.pinot.tools.admin.command.QuickstartRunner;
import org.apache.pinot.tools.streams.AirlineDataStream;
import org.apache.pinot.tools.utils.KafkaStarterUtils;
+import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
import static org.apache.pinot.tools.Quickstart.printStatus;
@@ -129,10 +131,10 @@ public class HybridQuickstart {
startKafka();
printStatus(Color.YELLOW, "***** Starting Zookeeper, 2 servers, 2 brokers and 1 controller *****");
runner.startAll();
- printStatus(Color.YELLOW, "***** Creating a server tenant with name 'airline' *****");
- runner.createServerTenantWith(1, 1, "airline");
- printStatus(Color.YELLOW, "***** Creating a broker tenant with name 'airline_broker' *****");
- runner.createBrokerTenantWith(2, "airline_broker");
+ printStatus(Color.YELLOW, "***** Creating a server tenant with name 'DefaultTenant' *****");
+ runner.createServerTenantWith(1, 1, TagNameUtils.DEFAULT_TENANT_NAME);
+ printStatus(Color.YELLOW, "***** Creating a broker tenant with name 'DefaultTenant' *****");
+ runner.createBrokerTenantWith(2, TagNameUtils.DEFAULT_TENANT_NAME);
printStatus(Color.YELLOW, "***** Adding airlineStats offline and realtime table *****");
runner.addTable();
printStatus(Color.YELLOW, "***** Launch data ingestion job to build index segments for airlineStats and push to controller *****");
@@ -155,7 +157,38 @@ public class HybridQuickstart {
printStatus(Color.YELLOW, "***** 7. Built and pushed an offline segment *****");
printStatus(Color.YELLOW,
"***** 8. Started publishing a Kafka stream for the realtime instance to start consuming *****");
- printStatus(Color.YELLOW, "***** go to http://localhost:9000/query to run a few queries *****");
+
+ String q1 = "select count(*) from airlineStats limit 10";
+ printStatus(Color.YELLOW, "Total number of documents in the table");
+ printStatus(Color.CYAN, "Query : " + q1);
+ printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q1)));
+ printStatus(Color.GREEN, "***************************************************");
+
+ String q2 = "select sum(Cancelled) from airlineStats group by AirlineID limit 5";
+ printStatus(Color.YELLOW, "Top 5 airlines in cancellation ");
+ printStatus(Color.CYAN, "Query : " + q2);
+ printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q2)));
+ printStatus(Color.GREEN, "***************************************************");
+
+ String q3 = "select sum(Flights) from airlineStats where Year > 2010 group by AirlineID, Year limit 5";
+ printStatus(Color.YELLOW, "Top 5 airlines in number of flights after 2010");
+ printStatus(Color.CYAN, "Query : " + q3);
+ printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
+ printStatus(Color.GREEN, "***************************************************");
+
+ String q4 = "select max(Flights) from airlineStats group by OriginCityName limit 5";
+ printStatus(Color.YELLOW, "Top 5 cities for number of flights");
+ printStatus(Color.CYAN, "Query : " + q4);
+ printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q4)));
+ printStatus(Color.GREEN, "***************************************************");
+
+ String q5 = "select AirlineID, OriginCityName, DestCityName, Year from airlineStats order by Year limit 5";
+ printStatus(Color.YELLOW, "Print AirlineID, OriginCityName, DestCityName, Year for 5 records ordered by Year");
+ printStatus(Color.CYAN, "Query : " + q5);
+ printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q5)));
+ printStatus(Color.GREEN, "***************************************************");
+
+ printStatus(Color.GREEN, "You can always go to http://localhost:9000/query to play around in the query console");
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
diff --git a/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_offline_table_config.json b/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_offline_table_config.json
index de73b4c..b91aa33 100644
--- a/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_offline_table_config.json
+++ b/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_offline_table_config.json
@@ -8,10 +8,7 @@
"segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
"replication": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP"
},
diff --git a/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json
index 6a893e4..cb5d690 100644
--- a/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json
+++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_realtime_table_config.json
@@ -12,10 +12,7 @@
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
diff --git a/pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json
index f483e85..0c32339 100644
--- a/pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json
+++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/docker/airlineStats_realtime_table_config.json
@@ -12,10 +12,7 @@
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
diff --git a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
index f5c4652..758e638 100644
--- a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
+++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_0.9/airlineStats_realtime_table_config.json
@@ -12,10 +12,7 @@
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
diff --git a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json
index 6a893e4..cb5d690 100644
--- a/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json
+++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/kafka_2.0/airlineStats_realtime_table_config.json
@@ -12,10 +12,7 @@
"replication": "1",
"replicasPerPartition": "1"
},
- "tenants": {
- "broker": "airline_broker",
- "server": "airline"
- },
+ "tenants": {},
"tableIndexConfig": {
"loadMode": "MMAP",
"streamConfigs": {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 03/06: Fix default value for dictionary byte
column (#5123)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e66df48e449b025ff42e98c1b06a6090be7c27b7
Author: Jialiang Li <jl...@linkedin.com>
AuthorDate: Fri Mar 6 17:56:52 2020 -0800
Fix default value for dictionary byte column (#5123)
* Fix default value for no dictionary byte column
---
.../apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java | 6 +++---
.../main/java/org/apache/pinot/core/minion/RawIndexConverter.java | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index 2698855..0dc772a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -208,10 +208,10 @@ public class MutableSegmentImpl implements MutableSegment {
// each forward index entry will contain a 4 byte dictionary ID
forwardIndexColumnSize = FieldSpec.DataType.INT.size();
int dictionaryColumnSize;
- if (dataType == FieldSpec.DataType.STRING) {
- dictionaryColumnSize = _statsHistory.getEstimatedAvgColSize(column);
- } else {
+ if (isFixedWidthColumn) {
dictionaryColumnSize = dataType.size();
+ } else {
+ dictionaryColumnSize = _statsHistory.getEstimatedAvgColSize(column);
}
// NOTE: preserve 10% buffer for cardinality to reduce the chance of re-sizing the dictionary
int estimatedCardinality = (int) (_statsHistory.getEstimatedCardinality(column) * 1.1);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
index 64baa46..9d5a484 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
@@ -165,10 +165,10 @@ public class RawIndexConverter {
// In bits
int lengthOfEachEntry;
- if (dataType.equals(FieldSpec.DataType.STRING)) {
- lengthOfEachEntry = columnMetadata.getColumnMaxLength() * Byte.SIZE;
- } else {
+ if (dataType.isFixedWidth()) {
lengthOfEachEntry = dataType.size() * Byte.SIZE;
+ } else {
+ lengthOfEachEntry = columnMetadata.getColumnMaxLength() * Byte.SIZE;
}
long dictionaryBasedIndexSize =
(long) numTotalDocs * columnMetadata.getBitsPerElement() + (long) cardinality * lengthOfEachEntry;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/06: Bugfixing the issue for ThreadLocal
DocIdSet in ExpressionFilterOperator (#5114)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 6f00be37ff467ddb163a6836c6ab6fb478072a60
Author: Jialiang Li <jl...@linkedin.com>
AuthorDate: Thu Mar 5 13:24:19 2020 -0800
Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator (#5114)
* Bugfixing the issue for ThreadLocal DocIdSet in ExpressionFilterOperator
* Update bitmap range start/end id
* Address comments
---
.../org/apache/pinot/core/operator/DocIdSetOperator.java | 8 +++++++-
.../pinot/core/operator/filter/ExpressionFilterOperator.java | 7 ++++---
.../integration/tests/BaseClusterIntegrationTestSet.java | 12 ++++++++++++
3 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
index 8528ef9..5db7db4 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/DocIdSetOperator.java
@@ -49,11 +49,17 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
private FilterBlockDocIdSet _filterBlockDocIdSet;
private BlockDocIdIterator _blockDocIdIterator;
private int _currentDocId = 0;
+ private boolean _threadLocal = true;
public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet) {
+ this(filterOperator, maxSizeOfDocIdSet, true);
+ }
+
+ public DocIdSetOperator(@Nonnull BaseFilterOperator filterOperator, int maxSizeOfDocIdSet, boolean threadLocal) {
Preconditions.checkArgument(maxSizeOfDocIdSet > 0 && maxSizeOfDocIdSet <= DocIdSetPlanNode.MAX_DOC_PER_CALL);
_filterOperator = filterOperator;
_maxSizeOfDocIdSet = maxSizeOfDocIdSet;
+ _threadLocal = threadLocal;
}
@Override
@@ -69,7 +75,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
}
int pos = 0;
- int[] docIds = THREAD_LOCAL_DOC_IDS.get();
+ int[] docIds = _threadLocal? THREAD_LOCAL_DOC_IDS.get(): new int[_maxSizeOfDocIdSet];
for (int i = 0; i < _maxSizeOfDocIdSet; i++) {
_currentDocId = _blockDocIdIterator.next();
if (_currentDocId == Constants.EOF) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
index 1f7a6d3..9078403 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExpressionFilterOperator.java
@@ -151,7 +151,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
throw new UnsupportedOperationException("Filter on expressions that return multi-values is not yet supported");
}
_blockDocIdIterator.setStartDocId(0);
- _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs - 1);
+ _blockDocIdIterator.setEndDocId(_expressionFilterOperator._numDocs);
}
@Override
@@ -223,8 +223,9 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
_currentBlockEndDocId = _currentBlockStartDocId + DocIdSetPlanNode.MAX_DOC_PER_CALL;
_currentBlockEndDocId = Math.min(_currentBlockEndDocId, _endDocId);
MutableRoaringBitmap bitmapRange = new MutableRoaringBitmap();
- bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId + 1);
+ bitmapRange.add(_currentBlockStartDocId, _currentBlockEndDocId);
MutableRoaringBitmap matchedBitmap = evaluate(bitmapRange);
+
_intIterator = matchedBitmap.getIntIterator();
_numDocsScanned += (_currentBlockEndDocId - _currentBlockStartDocId);
}
@@ -290,7 +291,7 @@ public class ExpressionFilterOperator extends BaseFilterOperator {
private MutableRoaringBitmap evaluate(MutableRoaringBitmap answer) {
BaseFilterOperator filterOperator = new BitmapWrappedFilterOperator(answer);
- DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL);
+ DocIdSetOperator docIdSetOperator = new DocIdSetOperator(filterOperator, DocIdSetPlanNode.MAX_DOC_PER_CALL, false);
ProjectionOperator projectionOperator =
new ProjectionOperator(_expressionFilterOperator._dataSourceMap, docIdSetOperator);
TransformOperator operator =
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
index 52b6fcb..52df943 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
@@ -170,6 +170,18 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati
String query;
List<String> h2queries;
query =
+ "SELECT COUNT(*) FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE CarrierDelay=15 AND ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT COUNT(*) FROM mytable WHERE ArrDelay > CarrierDelay LIMIT 1";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
+ "SELECT ArrDelay, CarrierDelay, (ArrDelay - CarrierDelay) AS diff FROM mytable WHERE ArrDelay > CarrierDelay ORDER BY diff, ArrDelay, CarrierDelay LIMIT 100000";
+ testSqlQuery(query, Collections.singletonList(query));
+ query =
"SELECT count(*) FROM mytable WHERE AirlineID > 20355 AND OriginState BETWEEN 'PA' AND 'DE' AND DepTime <> 2202 LIMIT 21";
testSqlQuery(query, Collections.singletonList(query));
query =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 05/06: Always copy test data out from jar file
(#5119)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch release-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 23b5c4fe200fb1d44812184f84881884642341d5
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Fri Mar 6 20:26:19 2020 -0800
Always copy test data out from jar file (#5119)
* Always copy test data out from jar file
* Address comments
---
.travis_quickstart.sh | 49 ++++++++++++-
.../org/apache/pinot/tools/HybridQuickstart.java | 3 +
.../tools/admin/command/QuickstartRunner.java | 10 ++-
.../org/apache/pinot/tools/utils/JarUtils.java | 83 ++++++++++++++++++++++
4 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/.travis_quickstart.sh b/.travis_quickstart.sh
index 2158b61..fcfb1b0 100755
--- a/.travis_quickstart.sh
+++ b/.travis_quickstart.sh
@@ -74,7 +74,52 @@ if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}
echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
exit 1
fi
-# kill $PID
-# sleep 30
+kill $PID
+sleep 30
+
+cd bin
+# Test quick-start-batch
+./quick-start-batch.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from baseballStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES}" -ne 97889 ]; then
+ echo 'Batch Quickstart: Incorrect result for count star query.'
+ exit 1
+fi
+kill $PID
+sleep 30
+
+# Test quick-start-streaming
+./quick-start-streaming.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then
+ echo 'Streaming Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
+ exit 1
+fi
+kill $PID
+sleep 30
+
+# Test quick-start-hybrid
+./quick-start-hybrid.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then
+ echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
+ exit 1
+fi
+kill $PID
+sleep 30
exit 0
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 7951a3f..4ab2aeb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -157,6 +157,9 @@ public class HybridQuickstart {
printStatus(Color.YELLOW, "***** 7. Built and pushed an offline segment *****");
printStatus(Color.YELLOW,
"***** 8. Started publishing a Kafka stream for the realtime instance to start consuming *****");
+ printStatus(Color.YELLOW,
+ "***** 9. Sleep 5 Seconds to wait for all components brought up *****");
+ Thread.sleep(5000);
String q1 = "select count(*) from airlineStats limit 10";
printStatus(Color.YELLOW, "Total number of documents in the table");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index bdbbaa6..0858db3 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -35,6 +35,7 @@ import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.QuickstartTableRequest;
+import org.apache.pinot.tools.utils.JarUtils;
import org.yaml.snakeyaml.Yaml;
@@ -178,7 +179,14 @@ public class QuickstartRunner {
String inputDirURI = spec.getInputDirURI();
if (!new File(inputDirURI).exists()) {
URL resolvedInputDirURI = QuickstartRunner.class.getClassLoader().getResource(inputDirURI);
- spec.setInputDirURI(resolvedInputDirURI.toURI().toString());
+ if (resolvedInputDirURI.getProtocol().equals("jar")) {
+ String[] splits = resolvedInputDirURI.getFile().split("!");
+ String inputDir = new File(_tempDir, "inputData").toString();
+ JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
+ spec.setInputDirURI(inputDir);
+ } else {
+ spec.setInputDirURI(resolvedInputDirURI.toString());
+ }
}
IngestionJobLauncher.runIngestionJob(spec);
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java
new file mode 100644
index 0000000..b850e1a
--- /dev/null
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Enumeration;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+
+public class JarUtils {
+ private static final String JAR_PREFIX = "jar:";
+ private static final String FILE_PREFIX = "file:";
+
+ private JarUtils() {
+ } // non-instantiable
+
+ /**
+ * Copies a directory from a jar file to an external directory.
+ */
+ public static void copyResourcesToDirectory(String fromJarFilePath, String jarDir, String destDir)
+ throws IOException {
+ if (fromJarFilePath.startsWith(JAR_PREFIX)) {
+ fromJarFilePath = fromJarFilePath.substring(JAR_PREFIX.length());
+ }
+ if (fromJarFilePath.startsWith(FILE_PREFIX)) {
+ fromJarFilePath = fromJarFilePath.substring(FILE_PREFIX.length());
+ }
+ JarFile fromJar = new JarFile(fromJarFilePath);
+ for (Enumeration<JarEntry> entries = fromJar.entries(); entries.hasMoreElements(); ) {
+ JarEntry entry = entries.nextElement();
+ if (entry.getName().startsWith(jarDir + "/") && !entry.isDirectory()) {
+ File dest = new File(destDir + "/" + entry.getName().substring(jarDir.length() + 1));
+ File parent = dest.getParentFile();
+ if (parent != null) {
+ parent.mkdirs();
+ }
+
+ FileOutputStream out = new FileOutputStream(dest);
+ InputStream in = fromJar.getInputStream(entry);
+
+ try {
+ byte[] buffer = new byte[8 * 1024];
+
+ int s = 0;
+ while ((s = in.read(buffer)) > 0) {
+ out.write(buffer, 0, s);
+ }
+ } catch (IOException e) {
+ throw new IOException("Could not copy asset from jar file", e);
+ } finally {
+ try {
+ in.close();
+ } catch (IOException ignored) {
+ }
+ try {
+ out.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org