You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/03/23 10:14:45 UTC
[01/50] [abbrv] kylin git commit: KYLIN-1383 remove deploy.env from
front end,
for permission control only depend on acl.defaultRole and acl.adminRole in
kylin.properties
Repository: kylin
Updated Branches:
refs/heads/1.5.x-HBase1.1.3 [created] dffbbf3bb
KYLIN-1383 remove deploy.env from front end, for permission control only depend on acl.defaultRole and acl.adminRole in kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d4922dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d4922dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d4922dc
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 2d4922dc6a55aa9376d424f1fce5454f7151e3de
Parents: c4d94f7
Author: Jason <ji...@163.com>
Authored: Thu Mar 3 14:24:38 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu Mar 3 14:25:32 2016 +0800
----------------------------------------------------------------------
webapp/app/js/services/kylinProperties.js | 1 +
webapp/app/partials/jobs/jobs.html | 2 +-
webapp/app/partials/models/models.html | 2 +-
webapp/app/partials/projects/projects.html | 2 +-
webapp/app/partials/query/query.html | 2 +-
5 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2d4922dc/webapp/app/js/services/kylinProperties.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js
index 546db2b..68e8766 100644
--- a/webapp/app/js/services/kylinProperties.js
+++ b/webapp/app/js/services/kylinProperties.js
@@ -55,6 +55,7 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
return false;
}
+ //deprecated
this.getDeployEnv = function () {
this.deployEnv = this.getProperty("deploy.env");
if (!this.deployEnv) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2d4922dc/webapp/app/partials/jobs/jobs.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/jobs/jobs.html b/webapp/app/partials/jobs/jobs.html
index daf4578..cc5840b 100644
--- a/webapp/app/partials/jobs/jobs.html
+++ b/webapp/app/partials/jobs/jobs.html
@@ -22,7 +22,7 @@
<form ng-if="userService.isAuthorized()">
<div class="form-group" ng-if="userService.hasRole('ROLE_MODELER')" >
<a class="btn btn-xs btn-info" href="projects" tooltip="Manage Project"><i class="fa fa-gears"></i></a>
- <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_ADMIN')||kylinConfig.getDeployEnv()!=='PROD'" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
+ <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_MODELER')" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
<i class="fa fa-plus"></i>
</a>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2d4922dc/webapp/app/partials/models/models.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/models.html b/webapp/app/partials/models/models.html
index 88cc6f3..c8e6ed1 100644
--- a/webapp/app/partials/models/models.html
+++ b/webapp/app/partials/models/models.html
@@ -21,7 +21,7 @@
<form class="navbar-form navbar-left" style="margin-top: 0px !important;" ng-if="userService.isAuthorized()">
<div class="form-group" ng-if="userService.hasRole('ROLE_MODELER')" >
<a class="btn btn-xs btn-info" href="projects" tooltip="Manage Project"><i class="fa fa-gears"></i></a>
- <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_ADMIN')||kylinConfig.getDeployEnv()!=='PROD'" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
+ <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_MODELER')" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
<i class="fa fa-plus"></i>
</a>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2d4922dc/webapp/app/partials/projects/projects.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/projects/projects.html b/webapp/app/partials/projects/projects.html
index 26a2037..96e4a91 100644
--- a/webapp/app/partials/projects/projects.html
+++ b/webapp/app/partials/projects/projects.html
@@ -17,7 +17,7 @@
-->
<div class="page-header">
- <button class="btn btn-primary btn-sm" ng-if="userService.hasRole('ROLE_ADMIN')||userService.hasRole('ROLE_MODELER')&&kylinConfig.getDeployEnv()!=='PROD'" ng-click="toCreateProj()"><i class="fa fa-plus"></i> Project</button>
+ <button class="btn btn-primary btn-sm" ng-if="userService.hasRole('ROLE_MODELER')" ng-click="toCreateProj()"><i class="fa fa-plus"></i> Project</button>
</div>
<div ng-if="!loading && projects.length == 0">
http://git-wip-us.apache.org/repos/asf/kylin/blob/2d4922dc/webapp/app/partials/query/query.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query.html b/webapp/app/partials/query/query.html
index 0af08d6..69fe649 100644
--- a/webapp/app/partials/query/query.html
+++ b/webapp/app/partials/query/query.html
@@ -21,7 +21,7 @@
<form class="navbar-form navbar-left" style="margin-top: 0px !important;" ng-if="userService.isAuthorized()">
<div class="form-group" ng-if="userService.hasRole('ROLE_MODELER')">
<a class="btn btn-xs btn-info" href="projects" tooltip="Manage Project"><i class="fa fa-gears"></i></a>
- <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_ADMIN')||kylinConfig.getDeployEnv()!=='PROD'" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
+ <a class="btn btn-xs btn-primary" ng-if="userService.hasRole('ROLE_MODELER')" style="width: 29px" tooltip="Add Project" ng-click="toCreateProj()">
<i class="fa fa-plus"></i>
</a>
</div>
[25/50] [abbrv] kylin git commit: Merge branch 'master' into
2.x-staging
Posted by sh...@apache.org.
Merge branch 'master' into 2.x-staging
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4e48f2ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4e48f2ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4e48f2ae
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 4e48f2ae63b66ebb04147f966fb72008bb939dff
Parents: c4be546 13e6a43
Author: Li Yang <li...@apache.org>
Authored: Thu Mar 10 15:42:41 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Mar 10 15:42:41 2016 +0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[32/50] [abbrv] kylin git commit: fix Linkage Error due to duplicate
loaded protobuf jar
Posted by sh...@apache.org.
fix Linkage Error due to duplicate loaded protobuf jar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6836815b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6836815b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6836815b
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 6836815b921c9a7a29dc9e44c34a70f7823b42ca
Parents: 376d695
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 11 12:12:21 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 11 12:12:26 2016 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 6 ++++++
jdbc/pom.xml | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6836815b/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index ba5ab1a..93f52b7 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -41,6 +41,12 @@
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6836815b/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index dae9d1e..805f34f 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -44,6 +44,12 @@
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
[50/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.1 API
Posted by sh...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.1 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dffbbf3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dffbbf3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dffbbf3b
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: dffbbf3bb4ff4414a751d5635974362513b36513
Parents: 5bdba62
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Mar 23 17:07:05 2016 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/invertedindex/IITest.java | 10 ++-
.../apache/kylin/common/KylinConfigBase.java | 3 +
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../test_case_data/sandbox/kylin_job_conf.xml | 86 +++++++++---------
examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
.../kylin/provision/BuildCubeWithEngine.java | 9 +-
.../kylin/provision/BuildIIWithStream.java | 13 +--
.../hbase/ii/ITInvertedIndexHBaseTest.java | 9 +-
pom.xml | 18 ++--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 36 +++-----
.../apache/kylin/rest/service/QueryService.java | 21 +++--
.../apache/kylin/rest/service/UserService.java | 16 ++--
.../kylin/storage/hbase/HBaseConnection.java | 36 ++++----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +-
.../hbase/cube/v1/HBaseClientKVIterator.java | 11 +--
.../hbase/cube/v1/RegionScannerAdapter.java | 11 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 15 +++-
.../observer/ObserverAggregationCache.java | 15 ++--
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 11 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/ii/IICreateHTableJob.java | 13 +--
.../hbase/ii/InvertedIndexStorageQuery.java | 6 +-
.../endpoint/EndpointTupleIterator.java | 13 +--
.../ii/coprocessor/endpoint/IIEndpoint.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 11 ++-
.../storage/hbase/steps/DeprecatedGCStep.java | 25 +++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../hbase/steps/HBaseStreamingOutput.java | 9 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 25 +++---
.../storage/hbase/util/CleanHtableCLI.java | 15 ++--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 18 ++--
.../hbase/util/DeployCoprocessorCLI.java | 23 ++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 19 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 42 ++++-----
.../kylin/storage/hbase/util/HBaseUsage.java | 10 ++-
.../storage/hbase/util/HbaseStreamingInput.java | 32 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 10 ++-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 18 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 16 ++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 31 +++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
58 files changed, 502 insertions(+), 545 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..5d2cfc4 100644
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.FIFOIterable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
@@ -222,6 +223,11 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
+ public int getBatch() {
+ return -1;
+ }
+
+ @Override
public boolean nextRaw(List<Cell> result) throws IOException {
if (iiRowIterator.hasNext()) {
IIRow iiRow = iiRowIterator.next();
@@ -233,7 +239,7 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
throw new NotImplementedException();
}
@@ -243,7 +249,7 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e3a73e8..38abc9a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -579,4 +579,7 @@ public class KylinConfigBase implements Serializable {
return getOptional("kylin.hive.beeline.params", "");
}
+ public String getPatchedFuzzyRowFilterVersion() {
+ return this.getOptional("kylin.hbase.filter.fuzzy.row.filter.version", "1.1.3");
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index bd947af..81704f2 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -1,20 +1,18 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
-->
+
<configuration>
<property>
@@ -26,44 +24,41 @@
</description>
</property>
- <property>
- <name>mapreduce.map.maxattempts</name>
- <value>2</value>
- </property>
+ <!-- uncomment the following 5 properties to enable lzo compressing
- <!--
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
- <property>
- <name>mapred.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
+ <property>
+ <name>mapred.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
--->
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ !-->
<property>
<name>mapreduce.job.max.split.locations</name>
@@ -76,5 +71,4 @@
<value>2</value>
<description>Block replication</description>
</property>
-
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 18f6feb..ff1c7eb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -28,12 +28,12 @@
<property>
<name>mapred.job.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
<name>mapred.job.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -58,7 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,10 +81,9 @@
<value>false</value>
</property>
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
+ <value>0.05</value>
</property>
<property>
@@ -114,7 +113,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -124,7 +123,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -169,7 +168,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -219,7 +218,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -234,7 +233,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -244,7 +243,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cfefef3..0446443 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -37,8 +37,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
@@ -60,6 +59,7 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.model.IEngineAware;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
@@ -367,10 +367,9 @@ public class BuildCubeWithEngine {
}
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
String tableName = segment.getStorageLocationIdentifier();
- HTable table = new HTable(conf, tableName);
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index 9b7cd14..3616ff9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -45,7 +45,8 @@ import java.util.TimeZone;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.util.ToolRunner;
@@ -200,7 +201,7 @@ public class BuildIIWithStream {
}
}
final IISegment segment = createSegment(iiName);
- final HTableInterface htable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(segment.getStorageLocationIdentifier());
+ final Table htable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(segment.getStorageLocationIdentifier()));
String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
ToolRunner.run(new IICreateHTableJob(), args);
@@ -238,7 +239,7 @@ public class BuildIIWithStream {
}
}
- private void build(SliceBuilder sliceBuilder, StreamingBatch batch, HTableInterface htable) throws IOException {
+ private void build(SliceBuilder sliceBuilder, StreamingBatch batch, Table htable) throws IOException {
final Slice slice = sliceBuilder.buildSlice(batch);
try {
loadToHBase(htable, slice, new IIKeyValueCodec(slice.getInfo()));
@@ -247,17 +248,17 @@ public class BuildIIWithStream {
}
}
- private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
+ private void loadToHBase(Table hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
List<Put> data = Lists.newArrayList();
for (IIRow row : codec.encodeKeyValue(slice)) {
final byte[] key = row.getKey().get();
final byte[] value = row.getValue().get();
Put put = new Put(key);
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
+ put.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
final ImmutableBytesWritable dictionary = row.getDictionary();
final byte[] dictBytes = dictionary.get();
if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) {
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
+ put.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
} else {
throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
index b5be703..d0ec67f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
@@ -20,9 +20,7 @@ package org.apache.kylin.storage.hbase.ii;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
@@ -49,7 +47,7 @@ public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
IIInstance ii;
IISegment seg;
- HConnection hconn;
+ Connection hconn;
TableRecordInfo info;
@@ -60,8 +58,7 @@ public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
this.seg = ii.getFirstSegment();
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- hconn = HConnectionManager.createConnection(hconf);
+ hconn = HBaseConnection.get(getTestConfig().getStorageUrl());
this.info = new TableRecordInfo(seg);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f878d8b..fd72780 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,12 +45,12 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<zookeeper.version>3.4.6</zookeeper.version>
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
- <hbase-hadoop2.version>0.98.4-hadoop2</hbase-hadoop2.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.8.1</kafka.version>
<!-- Dependency versions -->
@@ -62,6 +62,7 @@
<!-- Commons -->
<commons-cli.version>1.2</commons-cli.version>
+ <commons-codec.version>1.4</commons-codec.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.1</commons-lang3.version>
<commons-collections.version>3.2.1</commons-collections.version>
@@ -101,7 +102,7 @@
<calcite.version>1.6.0</calcite.version>
<!-- Curator.version Version -->
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
@@ -322,6 +323,11 @@
<version>${commons-cli.version}</version>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons-codec.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 2220113..0e71cd5 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -37,6 +37,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index 193f5a8..933c49d 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.UserService;
@@ -29,8 +29,8 @@ import org.apache.kylin.rest.service.UserService;
*/
public class MockAclHBaseStorage implements AclHBaseStorage {
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
@@ -49,7 +49,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
return mockedAclTable;
} else if (StringUtils.equals(tableName, userTableName)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 5a48e83..d40bdf3 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.UserService;
@@ -56,11 +57,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/AclService.java b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
index 58e093c..a03ff5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -225,17 +225,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -249,7 +248,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -265,7 +264,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -283,7 +281,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -294,17 +292,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 0c57d00..513311a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,8 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -441,35 +440,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
-
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- if (null != table) {
- table.close();
- }
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bf371be..10769e3 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -44,8 +44,9 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
@@ -128,14 +129,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -161,14 +161,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -180,9 +179,9 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/UserService.java b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index e4e2de3..624c49c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -29,7 +29,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -69,7 +69,7 @@ public class UserService implements UserManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -100,16 +100,15 @@ public class UserService implements UserManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
byte[] userAuthorities = serialize(user.getAuthorities());
htable = aclHBaseStorage.getTable(userTableName);
Put put = new Put(Bytes.toBytes(user.getUsername()));
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), userAuthorities);
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), userAuthorities);
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -119,14 +118,13 @@ public class UserService implements UserManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -141,7 +139,7 @@ public class UserService implements UserManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -161,7 +159,7 @@ public class UserService implements UserManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<String> authorities = new ArrayList<String>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 661e8e4..7452ad0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -49,13 +49,13 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> ConfigCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> ConnPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> ConnPool = new ConcurrentHashMap<String, Connection>();
static {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
- for (HConnection conn : ConnPool.values()) {
+ for (Connection conn : ConnPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -121,9 +121,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = ConfigCache.get(url);
if (conf == null) {
@@ -131,13 +131,13 @@ public class HBaseConnection {
ConfigCache.put(url, conf);
}
- HConnection connection = ConnPool.get(url);
+ Connection connection = ConnPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
ConnPool.put(url, connection);
}
@@ -156,8 +156,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -177,8 +177,8 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void createHTableIfNeeded(Connection conn, String tableName, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (tableExists(conn, tableName)) {
@@ -205,8 +205,8 @@ public class HBaseConnection {
}
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -216,10 +216,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 6d77240..9420d41 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -33,10 +33,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -83,7 +84,7 @@ public class HBaseResourceStore extends ResourceStore {
// final Map<String, String> tableNameMap; // path prefix ==> HBase table name
- private HConnection getConnection() throws IOException {
+ private Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -126,7 +127,7 @@ public class HBaseResourceStore extends ResourceStore {
ArrayList<String> result = new ArrayList<String>();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
scan.setFilter(new KeyOnlyFilter());
try {
@@ -171,7 +172,7 @@ public class HBaseResourceStore extends ResourceStore {
}
tuneScanParameters(scan);
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
List<RawResource> result = Lists.newArrayList();
try {
ResultScanner scanner = table.getScanner(scan);
@@ -253,13 +254,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -267,7 +267,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -280,8 +280,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -290,11 +288,10 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -319,7 +316,7 @@ public class HBaseResourceStore extends ResourceStore {
get.addColumn(B_FAMILY, B_COLUMN_TS);
}
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
Result result = table.get(get);
boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
@@ -335,7 +332,7 @@ public class HBaseResourceStore extends ResourceStore {
return endRow;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -361,7 +358,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -369,8 +366,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 909de39..2bc8a3b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,8 +25,9 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 3d7f620..952e9af 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -148,7 +148,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
index 8aace22..54c399f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
@@ -24,8 +24,9 @@ import java.util.Iterator;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -40,14 +41,14 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
byte[] family;
- HTableInterface table;
+ Table table;
ResultScanner scanner;
Iterator<Result> iterator;
- public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family) throws IOException {
+ public HBaseClientKVIterator(Connection hconn, String tableName, byte[] family) throws IOException {
this.family = family;
- this.table = hconn.getTable(tableName);
+ this.table = hconn.getTable(TableName.valueOf(tableName));
this.scanner = table.getScanner(family);
this.iterator = scanner.iterator();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 6342c5c..25c2dda 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -23,9 +23,11 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +52,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +62,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +96,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index c7b650a..8dba1b1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index d8b61b3..635a32f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -24,7 +24,9 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -116,8 +118,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +128,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +162,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..1809a44 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -24,12 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +110,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +120,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +159,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index b8ac3d5..7435d07 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,7 +23,7 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e6f9ac1..902872f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -38,8 +38,9 @@ import javax.annotation.Nullable;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -258,7 +259,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequests.get(0).getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -310,7 +311,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
Map<byte[], CubeVisitProtos.CubeVisitResponse> results;
try {
- results = getResults(builder.build(), conn.getTable(cubeSeg.getStorageLocationIdentifier()), epRange.getFirst(), epRange.getSecond());
+ results = getResults(builder.build(), conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier())), epRange.getFirst(), epRange.getSecond());
} catch (Throwable throwable) {
throw new RuntimeException("Error when visiting cubes by endpoint:", throwable);
}
@@ -345,7 +346,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
- private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, HTableInterface table, byte[] startKey, byte[] endKey) throws Throwable {
+ private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, Table table, byte[] startKey, byte[] endKey) throws Throwable {
Map<byte[], CubeVisitProtos.CubeVisitResponse> results = table.coprocessorService(CubeVisitProtos.CubeVisitService.class, startKey, endKey, new Batch.Call<CubeVisitProtos.CubeVisitService, CubeVisitProtos.CubeVisitResponse>() {
public CubeVisitProtos.CubeVisitResponse call(CubeVisitProtos.CubeVisitService rowsService) throws IOException {
ServerRpcController controller = new ServerRpcController();
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index ef53cb7..2b1b512 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -25,8 +25,9 @@ import java.util.List;
import javax.annotation.Nullable;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -135,8 +136,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getPkStart(), scanRequest.getPkEnd(), scanRequest.getFuzzyKeys(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
[38/50] [abbrv] kylin git commit: KYLIN-1286 fix licenses in resource
files
Posted by sh...@apache.org.
KYLIN-1286 fix licenses in resource files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa49ece1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa49ece1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa49ece1
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: aa49ece1f2cf2a44b590cf2ac938378b3046995e
Parents: 0232f17
Author: Li Yang <li...@apache.org>
Authored: Fri Mar 11 15:37:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Mar 11 15:37:15 2016 +0800
----------------------------------------------------------------------
.travis.yml | 1 -
build/conf/kylin.properties | 18 ++++++++++++++++--
build/conf/kylin_hive_conf.xml | 17 ++++++++++++++++-
build/conf/kylin_job_conf.xml | 17 ++++++++++++++++-
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 17 +++++++++++++++++
.../test_case_data/localmeta/kylin.properties | 18 ++++++++++++++++--
examples/test_case_data/sandbox/kylin.properties | 17 ++++++++++++++++-
webapp/app/js/controllers/cubeDimensions.js | 4 ++--
webapp/app/js/controllers/cubeMeasures.js | 4 ++--
.../app/partials/tables/loadStreamingTable.html | 17 +++++++++++++++++
10 files changed, 118 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index f5c99a7..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1 +0,0 @@
-language: java
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index d694e9f..e87494d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -1,5 +1,19 @@
-## Config for Kylin Engine ##
-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
# optional information for the owner of kylin platform, it can be your team's email
# currently it will be attached to each kylin's htable attribute
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index f91f489..a11d768 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -1,4 +1,19 @@
-<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
<name>dfs.replication</name>
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 3ecaeda..19c8847 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -1,4 +1,19 @@
-<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
index bf21362..0f81d2e 100644
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
dir=$(dirname ${0})
cd ${dir}
cd ..
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 978102f..051a312 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -1,5 +1,19 @@
-## Config for Kylin Engine ##
-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
# optional information for the owner of kylin platform, it can be your team's email
# currently it will be attached to each kylin's htable attribute
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 04681af..f1ecd64 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -1,4 +1,19 @@
-## Config for Kylin Engine ##
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
# optional information for the owner of kylin platform, it can be your team's email
# currently it will be attached to each kylin's htable attribute
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index 1c92616..9e13dcc 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -7,14 +7,14 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/license$s/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
'use strict';
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 3292e06..f4bfab7 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -7,14 +7,14 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/license$s/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
'use strict';
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa49ece1/webapp/app/partials/tables/loadStreamingTable.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/loadStreamingTable.html b/webapp/app/partials/tables/loadStreamingTable.html
index da79369..fb771b0 100644
--- a/webapp/app/partials/tables/loadStreamingTable.html
+++ b/webapp/app/partials/tables/loadStreamingTable.html
@@ -1,3 +1,20 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
<div class="modal-body streaming-source" style="height: 360px;">
<div class="col-xs-5">
[41/50] [abbrv] kylin git commit: KYLIN-1286 fix java license files
Posted by sh...@apache.org.
KYLIN-1286 fix java license files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a44a57c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a44a57c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a44a57c0
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: a44a57c0c451469a0d42bdc87a5b7191a7dda6b6
Parents: d0e096a
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 11 16:20:51 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 11 16:20:51 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeCodeSystem.java | 18 ++++++++++++++++++
.../kylin/cube/gridtable/CubeGridTable.java | 18 ++++++++++++++++++
.../gridtable/CuboidToGridTableMapping.java | 18 ++++++++++++++++++
.../kylin/cube/inmemcubing/CuboidResult.java | 18 ++++++++++++++++++
.../cube/model/CubeJoinedFlatTableDesc.java | 18 ++++++++++++++++++
.../kylin/gridtable/DefaultGTComparator.java | 18 ++++++++++++++++++
.../kylin/gridtable/GTAggregateScanner.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/GTBuilder.java | 18 ++++++++++++++++++
.../apache/kylin/gridtable/GTFilterScanner.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/gridtable/GTInfo.java | 18 ++++++++++++++++++
.../apache/kylin/gridtable/GTInvertedIndex.java | 18 ++++++++++++++++++
.../gridtable/GTInvertedIndexOfColumn.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/GTRecord.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/GTRowBlock.java | 18 ++++++++++++++++++
.../kylin/gridtable/GTSampleCodeSystem.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/GTScanRange.java | 18 ++++++++++++++++++
.../kylin/gridtable/GTScanRangePlanner.java | 18 ++++++++++++++++++
.../apache/kylin/gridtable/GTScanRequest.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/gridtable/GTUtil.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/GridTable.java | 18 ++++++++++++++++++
.../apache/kylin/gridtable/IGTCodeSystem.java | 18 ++++++++++++++++++
.../apache/kylin/gridtable/IGTComparator.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/IGTScanner.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/IGTStore.java | 18 ++++++++++++++++++
.../org/apache/kylin/gridtable/IGTWriter.java | 18 ++++++++++++++++++
.../gridtable/memstore/GTSimpleMemStore.java | 18 ++++++++++++++++++
.../kylin/engine/IStreamingCubingEngine.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/job/lock/JobLock.java | 18 ++++++++++++++++++
.../org/apache/kylin/job/lock/MockJobLock.java | 18 ++++++++++++++++++
.../apache/kylin/job/BasicLocalMetaTest.java | 18 ++++++++++++++++++
.../kylin/storage/ICachableStorageQuery.java | 18 ++++++++++++++++++
.../cache/AbstractCacheFledgedQuery.java | 18 ++++++++++++++++++
.../storage/cache/CacheFledgedDynamicQuery.java | 18 ++++++++++++++++++
.../storage/cache/CacheFledgedStaticQuery.java | 18 ++++++++++++++++++
.../kylin/storage/cache/StreamSQLResult.java | 18 ++++++++++++++++++
.../storage/cache/TsConditionExtractor.java | 18 ++++++++++++++++++
.../apache/kylin/storage/util/SizeOfUtil.java | 18 ++++++++++++++++++
.../kylin/storage/cache/DynamicCacheTest.java | 18 ++++++++++++++++++
.../apache/kylin/storage/cache/EhcacheTest.java | 18 ++++++++++++++++++
.../kylin/storage/cache/StaticCacheTest.java | 18 ++++++++++++++++++
.../kylin/storage/cache/StorageMockUtils.java | 18 ++++++++++++++++++
.../kylin/storage/hybrid/HybridManagerTest.java | 18 ++++++++++++++++++
.../storage/translate/ColumnValueRangeTest.java | 18 ++++++++++++++++++
.../kylin/engine/mr/ByteArrayWritable.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/engine/mr/MRUtil.java | 18 ++++++++++++++++++
.../engine/mr/steps/BaseCuboidMapperBase.java | 18 ++++++++++++++++++
.../engine/mr/steps/CubingExecutableUtil.java | 18 ++++++++++++++++++
.../mr/steps/FactDistinctColumnPartitioner.java | 18 ++++++++++++++++++
.../mr/steps/FactDistinctColumnsMapperBase.java | 18 ++++++++++++++++++
.../engine/mr/steps/InMemCuboidMapper.java | 18 ++++++++++++++++++
.../engine/mr/steps/InMemCuboidReducer.java | 18 ++++++++++++++++++
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 18 ++++++++++++++++++
.../mr/steps/MapContextGTRecordWriter.java | 18 ++++++++++++++++++
.../steps/FactDistinctColumnsReducerTest.java | 18 ++++++++++++++++++
.../kylin/engine/streaming/BootstrapConfig.java | 18 ++++++++++++++++++
.../kylin/engine/streaming/cli/MonitorCLI.java | 18 ++++++++++++++++++
.../streaming/monitor/StreamingMonitor.java | 18 ++++++++++++++++++
.../model/IIKeyValueCodecWithState.java | 18 ++++++++++++++++++
.../measure/FixedPointLongCodecTest.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/jdbc/KylinClient.java | 18 ++++++++++++++++++
.../kylin/jdbc/KylinPreparedStatement.java | 18 ++++++++++++++++++
.../apache/kylin/jdbc/util/Log4jConfigurer.java | 18 ++++++++++++++++++
.../resources/org-apache-kylin-jdbc.properties | 18 ++++++++++++++++++
kylin-it/src/test/resources/logging.properties | 20 +++++++++++++++++++-
.../kylin/rest/security/AclHBaseStorage.java | 18 ++++++++++++++++++
.../security/KylinAuthenticationProvider.java | 18 ++++++++++++++++++
.../rest/security/MockAclHBaseStorage.java | 18 ++++++++++++++++++
.../apache/kylin/rest/security/MockHTable.java | 18 ++++++++++++++++++
.../rest/security/RealAclHBaseStorage.java | 18 ++++++++++++++++++
.../rest/security/SAMLUserDetailsService.java | 18 ++++++++++++++++++
.../source/kafka/config/KafkaClusterConfig.java | 18 ++++++++++++++++++
.../source/kafka/util/KafkaSampleProducer.java | 18 ++++++++++++++++++
.../kylin/source/kafka/util/KafkaUtils.java | 18 ++++++++++++++++++
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 18 ++++++++++++++++++
74 files changed, 1333 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f83f920..f15d3f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.cube.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 5f0bb07..47885f7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.cube.gridtable;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index d0fca02..aad37fd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.cube.gridtable;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CuboidResult.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CuboidResult.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CuboidResult.java
index c5a73b0..7d41faf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CuboidResult.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CuboidResult.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.cube.inmemcubing;
import org.apache.kylin.gridtable.GridTable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index b773b3f..db1942b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.cube.model;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
index 714571f..c0fe939 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import org.apache.kylin.common.util.ByteArray;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index f8fe661..7356e77 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.Closeable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTBuilder.java
index 5eefa54..60de2d1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTBuilder.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.Closeable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index bf4ce8c..59f1df7 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index b6da738..55356f9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
index dd96539..a2c713a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
index 9487423..6069e73 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.util.NavigableMap;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b671c88..d09ab9a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java
index 049fa25..2cbbd8d 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRowBlock.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.DataInputStream;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index d68d7d4..eb232e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index 1ca2cd0..56f55f6 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 559a245..3fb4197 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 8b0779d..02ad820 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 7cf7f3c..b231d18 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
index 8f81654..b2923ec 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.Closeable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 3a22091..dbd5e41 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/IGTComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTComparator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTComparator.java
index 4edc4db..1560f1b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTComparator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTComparator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.util.Comparator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
index a82dffa..c0da569 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.Closeable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
index f4c44f8..fa5ce80 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/IGTWriter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTWriter.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTWriter.java
index 6733c68..845f03c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTWriter.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTWriter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable;
import java.io.Closeable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index 9675aa1..a1e81a8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.gridtable.memstore;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java b/core-job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
index 0359ce9..cec57a7 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine;
import org.apache.kylin.cube.CubeSegment;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-job/src/main/java/org/apache/kylin/job/lock/JobLock.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/lock/JobLock.java b/core-job/src/main/java/org/apache/kylin/job/lock/JobLock.java
index 490e3f7..bbfb801 100644
--- a/core-job/src/main/java/org/apache/kylin/job/lock/JobLock.java
+++ b/core-job/src/main/java/org/apache/kylin/job/lock/JobLock.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.job.lock;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java b/core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java
index f5cea4e..cac17b9 100644
--- a/core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java
+++ b/core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.job.lock;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java b/core-job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
index d034a73..56934e3 100644
--- a/core-job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.job;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java
index 179202e..f401f54 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/ICachableStorageQuery.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage;
import com.google.common.collect.Range;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index a5ca800..4b94cb6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import net.sf.ehcache.Cache;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
index 1481754..a93e746 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
index 57f33f6..45c13fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 74def9e..f6809df 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/cache/TsConditionExtractor.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/TsConditionExtractor.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/TsConditionExtractor.java
index 3a1b52c..f972910 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/TsConditionExtractor.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/TsConditionExtractor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import org.apache.kylin.common.util.DateFormat;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java b/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
index ccbe845..aa379d4 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.util;
import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
index 3193bbb..0adaff3 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
index 4422fbd..5a88f2f 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
index b1665df..0a4795f 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
index fb6e765..5f8f08f 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.cache;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 12fdd53..6cbcbc4 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hybrid;
import org.apache.kylin.common.util.JsonUtil;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
index d32f171..37f928d 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.translate;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
index 37a8841..a504899 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/ByteArrayWritable.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr;
import java.io.DataInput;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index 41c8b6b..d839c2c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr;
import org.apache.kylin.common.KylinConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 5fb9098..71c3cbf 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
index 71f27f2..ab4a1de 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
index 0320ebe..b3498ff 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index cc7b6df..c504f0d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index e42a6b5..bd07a0d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index e72c38b..ec36242 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
index e201705..ee65132 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index 6b4d07d..765f080 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index 928db74..ea4e38e 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.mr.steps;
import java.io.File;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
index a4c4618..e690e9a 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.streaming;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
index d7dc6b3..10b186f 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/MonitorCLI.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.streaming.cli;
import java.util.List;
[20/50] [abbrv] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6
(with Edward Zhang)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c184f0b..aed7c27 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -18,6 +18,7 @@
package org.apache.calcite.sql2rel;
+import org.apache.calcite.avatica.util.Spaces;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
@@ -25,26 +26,32 @@ import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptSamplingParameters;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.prepare.RelOptTableImpl;
import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.SingleRel;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Collect;
import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.Sample;
+import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.core.Uncollect;
import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalIntersect;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.logical.LogicalMinus;
@@ -56,6 +63,7 @@ import org.apache.calcite.rel.logical.LogicalTableScan;
import org.apache.calcite.rel.logical.LogicalUnion;
import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.stream.Delta;
import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -72,8 +80,8 @@ import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.rex.RexWindowBound;
import org.apache.calcite.schema.ModifiableTable;
import org.apache.calcite.schema.ModifiableView;
@@ -85,6 +93,7 @@ import org.apache.calcite.sql.SemiJoinType;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDelete;
import org.apache.calcite.sql.SqlDynamicParam;
@@ -102,10 +111,12 @@ import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlNumericLiteral;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSampleSpec;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.SqlSelectKeyword;
import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlUnnestOperator;
import org.apache.calcite.sql.SqlUpdate;
import org.apache.calcite.sql.SqlUtil;
import org.apache.calcite.sql.SqlValuesOperator;
@@ -138,8 +149,10 @@ import org.apache.calcite.sql.validate.SqlValidatorImpl;
import org.apache.calcite.sql.validate.SqlValidatorNamespace;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Litmus;
import org.apache.calcite.util.NlsString;
import org.apache.calcite.util.NumberUtil;
import org.apache.calcite.util.Pair;
@@ -158,15 +171,16 @@ import com.google.common.collect.Sets;
import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.util.AbstractList;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Deque;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.Stack;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -178,7 +192,7 @@ import static org.apache.calcite.util.Static.RESOURCE;
* OVERRIDE POINT:
* - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
* - isTrimUnusedFields(), override to false
- * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
+ * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
*/
/**
@@ -190,3851 +204,4825 @@ import static org.apache.calcite.util.Static.RESOURCE;
* {@link #convertExpression(SqlNode)}.
*/
public class SqlToRelConverter {
- //~ Static fields/initializers ---------------------------------------------
-
- protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
-
- private static final BigDecimal TWO = BigDecimal.valueOf(2L);
-
- //~ Instance fields --------------------------------------------------------
-
- protected final SqlValidator validator;
- protected final RexBuilder rexBuilder;
- protected final Prepare.CatalogReader catalogReader;
- protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
- protected final List<RelNode> leaves = new ArrayList<>();
- private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
- private final SqlOperatorTable opTab;
- private boolean shouldConvertTableAccess;
- protected final RelDataTypeFactory typeFactory;
- private final SqlNodeToRexConverter exprConverter;
- private boolean decorrelationEnabled;
- private boolean trimUnusedFields;
- private boolean shouldCreateValuesRel;
- private boolean isExplain;
- private int nDynamicParamsInExplain;
-
- /**
- * Fields used in name resolution for correlated subqueries.
- */
- private final Map<String, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
- private int nextCorrel = 0;
-
- private static final String CORREL_PREFIX = "$cor";
-
- /**
- * Stack of names of datasets requested by the <code>
- * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
- */
- private final Stack<String> datasetStack = new Stack<>();
-
- /**
- * Mapping of non-correlated subqueries that have been converted to their
- * equivalent constants. Used to avoid re-evaluating the subquery if it's
- * already been evaluated.
- */
- private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
-
- public final RelOptTable.ViewExpander viewExpander;
-
- //~ Constructors -----------------------------------------------------------
- /**
- * Creates a converter.
- *
- * @param viewExpander Preparing statement
- * @param validator Validator
- * @param catalogReader Schema
- * @param planner Planner
- * @param rexBuilder Rex builder
- * @param convertletTable Expression converter
- */
- @Deprecated
- // will be removed before 2.0
- public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable) {
- this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable);
- }
-
- /* Creates a converter. */
- public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable) {
- this.viewExpander = viewExpander;
- this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable();
- this.validator = validator;
- this.catalogReader = catalogReader;
- this.defaultValueFactory = new NullDefaultValueFactory();
- this.subqueryConverter = new NoOpSubqueryConverter();
- this.rexBuilder = cluster.getRexBuilder();
- this.typeFactory = rexBuilder.getTypeFactory();
- this.cluster = Preconditions.checkNotNull(cluster);
- this.shouldConvertTableAccess = true;
- this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
- decorrelationEnabled = true;
- trimUnusedFields = false;
- shouldCreateValuesRel = true;
- isExplain = false;
- nDynamicParamsInExplain = 0;
- }
-
- //~ Methods ----------------------------------------------------------------
+ //~ Static fields/initializers ---------------------------------------------
+
+ protected static final Logger SQL2REL_LOGGER =
+ CalciteTrace.getSqlToRelTracer();
+
+ private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
+ /** Size of the smallest IN list that will be converted to a semijoin to a
+ * static table. */
+ public static final int IN_SUBQUERY_THRESHOLD = 20;
+
+ //~ Instance fields --------------------------------------------------------
+
+ protected final SqlValidator validator;
+ protected final RexBuilder rexBuilder;
+ protected final Prepare.CatalogReader catalogReader;
+ protected final RelOptCluster cluster;
+ private DefaultValueFactory defaultValueFactory;
+ private SubqueryConverter subqueryConverter;
+ protected final List<RelNode> leaves = new ArrayList<>();
+ private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
+ private final SqlOperatorTable opTab;
+ private boolean shouldConvertTableAccess;
+ protected final RelDataTypeFactory typeFactory;
+ private final SqlNodeToRexConverter exprConverter;
+ private boolean decorrelationEnabled;
+ private boolean trimUnusedFields;
+ private boolean shouldCreateValuesRel;
+ private boolean isExplain;
+ private int nDynamicParamsInExplain;
+
+ /**
+ * Fields used in name resolution for correlated subqueries.
+ */
+ private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred =
+ new HashMap<>();
+
+ /**
+ * Stack of names of datasets requested by the <code>
+ * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
+ */
+ private final Deque<String> datasetStack = new ArrayDeque<>();
+
+ /**
+ * Mapping of non-correlated subqueries that have been converted to their
+ * equivalent constants. Used to avoid re-evaluating the subquery if it's
+ * already been evaluated.
+ */
+ private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
+ new HashMap<>();
+
+ public final RelOptTable.ViewExpander viewExpander;
+
+ /** Whether to expand sub-queries. If false, each sub-query becomes a
+ * {@link org.apache.calcite.rex.RexSubQuery}. */
+ private boolean expand = true;
+
+ //~ Constructors -----------------------------------------------------------
+ /**
+ * Creates a converter.
+ *
+ * @param viewExpander Preparing statement
+ * @param validator Validator
+ * @param catalogReader Schema
+ * @param planner Planner
+ * @param rexBuilder Rex builder
+ * @param convertletTable Expression converter
+ */
+ @Deprecated // will be removed before 2.0
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptPlanner planner,
+ RexBuilder rexBuilder,
+ SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader,
+ RelOptCluster.create(planner, rexBuilder), convertletTable);
+ }
+
+ /* Creates a converter. */
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable) {
+ this.viewExpander = viewExpander;
+ this.opTab =
+ (validator
+ == null) ? SqlStdOperatorTable.instance()
+ : validator.getOperatorTable();
+ this.validator = validator;
+ this.catalogReader = catalogReader;
+ this.defaultValueFactory = new NullDefaultValueFactory();
+ this.subqueryConverter = new NoOpSubqueryConverter();
+ this.rexBuilder = cluster.getRexBuilder();
+ this.typeFactory = rexBuilder.getTypeFactory();
+ this.cluster = Preconditions.checkNotNull(cluster);
+ this.shouldConvertTableAccess = true;
+ this.exprConverter =
+ new SqlNodeToRexConverterImpl(convertletTable);
+ decorrelationEnabled = true;
+ trimUnusedFields = false;
+ shouldCreateValuesRel = true;
+ isExplain = false;
+ nDynamicParamsInExplain = 0;
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ /**
+ * @return the RelOptCluster in use.
+ */
+ public RelOptCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Returns the row-expression builder.
+ */
+ public RexBuilder getRexBuilder() {
+ return rexBuilder;
+ }
+
+ /**
+ * Returns the number of dynamic parameters encountered during translation;
+ * this must only be called after {@link #convertQuery}.
+ *
+ * @return number of dynamic parameters
+ */
+ public int getDynamicParamCount() {
+ return dynamicParamSqlNodes.size();
+ }
+
+ /**
+ * Returns the type inferred for a dynamic parameter.
+ *
+ * @param index 0-based index of dynamic parameter
+ * @return inferred type, never null
+ */
+ public RelDataType getDynamicParamType(int index) {
+ SqlNode sqlNode = dynamicParamSqlNodes.get(index);
+ if (sqlNode == null) {
+ throw Util.needToImplement("dynamic param type inference");
+ }
+ return validator.getValidatedNodeType(sqlNode);
+ }
+
+ /**
+ * Returns the current count of the number of dynamic parameters in an
+ * EXPLAIN PLAN statement.
+ *
+ * @param increment if true, increment the count
+ * @return the current count before the optional increment
+ */
+ public int getDynamicParamCountInExplain(boolean increment) {
+ int retVal = nDynamicParamsInExplain;
+ if (increment) {
+ ++nDynamicParamsInExplain;
+ }
+ return retVal;
+ }
+
+ /**
+ * @return mapping of non-correlated subqueries that have been converted to
+ * the constants that they evaluate to
+ */
+ public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
+ return mapConvertedNonCorrSubqs;
+ }
+
+ /**
+ * Adds to the current map of non-correlated converted subqueries the
+ * elements from another map that contains non-correlated subqueries that
+ * have been converted by another SqlToRelConverter.
+ *
+ * @param alreadyConvertedNonCorrSubqs the other map
+ */
+ public void addConvertedNonCorrSubqs(
+ Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+ mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
+ }
+
+ /**
+ * Set a new DefaultValueFactory. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param factory new DefaultValueFactory
+ */
+ public void setDefaultValueFactory(DefaultValueFactory factory) {
+ defaultValueFactory = factory;
+ }
+
+ /**
+ * Sets a new SubqueryConverter. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param converter new SubqueryConverter
+ */
+ public void setSubqueryConverter(SubqueryConverter converter) {
+ subqueryConverter = converter;
+ }
+
+ /**
+ * Indicates that the current statement is part of an EXPLAIN PLAN statement
+ *
+ * @param nDynamicParams number of dynamic parameters in the statement
+ */
+ public void setIsExplain(int nDynamicParams) {
+ isExplain = true;
+ nDynamicParamsInExplain = nDynamicParams;
+ }
+
+ /**
+ * Controls whether table access references are converted to physical rels
+ * immediately. The optimizer doesn't like leaf rels to have
+ * {@link Convention#NONE}. However, if we are doing further conversion
+ * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
+ * defer conversion. To have any effect, this must be called before any
+ * convert method.
+ *
+ * @param enabled true for immediate conversion (the default); false to
+ * generate logical LogicalTableScan instances
+ */
+ public void enableTableAccessConversion(boolean enabled) {
+ shouldConvertTableAccess = enabled;
+ }
+
+ /**
+ * Controls whether instances of
+ * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
+ * may not be supported by all physical implementations. To have any effect,
+ * this must be called before any convert method.
+ *
+ * @param enabled true to allow LogicalValues to be generated (the default);
+ * false to force substitution of Project+OneRow instead
+ */
+ public void enableValuesRelCreation(boolean enabled) {
+ shouldCreateValuesRel = enabled;
+ }
+
+ private void checkConvertedType(SqlNode query, RelNode result) {
+ if (query.isA(SqlKind.DML)) {
+ return;
+ }
+ // Verify that conversion from SQL to relational algebra did
+ // not perturb any type information. (We can't do this if the
+ // SQL statement is something like an INSERT which has no
+ // validator type information associated with its result,
+ // hence the namespace check above.)
+ final List<RelDataTypeField> validatedFields =
+ validator.getValidatedNodeType(query).getFieldList();
+ final RelDataType validatedRowType =
+ validator.getTypeFactory().createStructType(
+ Pair.right(validatedFields),
+ SqlValidatorUtil.uniquify(Pair.left(validatedFields)));
+
+ final List<RelDataTypeField> convertedFields =
+ result.getRowType().getFieldList().subList(0, validatedFields.size());
+ final RelDataType convertedRowType =
+ validator.getTypeFactory().createStructType(convertedFields);
+
+ if (!RelOptUtil.equal("validated row type", validatedRowType,
+ "converted row type", convertedRowType, Litmus.IGNORE)) {
+ throw new AssertionError("Conversion to relational algebra failed to "
+ + "preserve datatypes:\n"
+ + "validated type:\n"
+ + validatedRowType.getFullTypeString()
+ + "\nconverted type:\n"
+ + convertedRowType.getFullTypeString()
+ + "\nrel:\n"
+ + RelOptUtil.toString(result));
+ }
+ }
+
+ public RelNode flattenTypes(
+ RelNode rootRel,
+ boolean restructure) {
+ RelStructuredTypeFlattener typeFlattener =
+ new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
+ return typeFlattener.rewrite(rootRel, restructure);
+ }
+
+ /**
+ * If subquery is correlated and decorrelation is enabled, performs
+ * decorrelation.
+ *
+ * @param query Query
+ * @param rootRel Root relational expression
+ * @return New root relational expression after decorrelation
+ */
+ public RelNode decorrelate(SqlNode query, RelNode rootRel) {
+ if (!enableDecorrelation()) {
+ return rootRel;
+ }
+ final RelNode result = decorrelateQuery(rootRel);
+ if (result != rootRel) {
+ checkConvertedType(query, result);
+ }
+ return result;
+ }
+
+ /**
+ * Walks over a tree of relational expressions, replacing each
+ * {@link RelNode} with a 'slimmed down' relational expression that projects
+ * only the fields required by its consumer.
+ *
+ * <p>This may make things easier for the optimizer, by removing crud that
+ * would expand the search space, but is difficult for the optimizer itself
+ * to do it, because optimizer rules must preserve the number and type of
+ * fields. Hence, this transform that operates on the entire tree, similar
+ * to the {@link RelStructuredTypeFlattener type-flattening transform}.
+ *
+ * <p>Currently this functionality is disabled in farrago/luciddb; the
+ * default implementation of this method does nothing.
+ *
+ * @param ordered Whether the relational expression must produce results in
+ * a particular order (typically because it has an ORDER BY at top level)
+ * @param rootRel Relational expression that is at the root of the tree
+ * @return Trimmed relational expression
+ */
+ public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) {
+ // Trim fields that are not used by their consumer.
+ if (isTrimUnusedFields()) {
+ final RelFieldTrimmer trimmer = newFieldTrimmer();
+ final List<RelCollation> collations =
+ rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
+ rootRel = trimmer.trim(rootRel);
+ if (!ordered
+ && collations != null
+ && !collations.isEmpty()
+ && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
+ final RelTraitSet traitSet = rootRel.getTraitSet()
+ .replace(RelCollationTraitDef.INSTANCE, collations);
+ rootRel = rootRel.copy(traitSet, rootRel.getInputs());
+ }
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(
+ RelOptUtil.dumpPlan(
+ "Plan after trimming unused fields",
+ rootRel,
+ false,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ }
+ }
+ return rootRel;
+ }
+
+ /**
+ * Creates a RelFieldTrimmer.
+ *
+ * @return Field trimmer
+ */
+ protected RelFieldTrimmer newFieldTrimmer() {
+ final RelBuilder relBuilder =
+ RelFactories.LOGICAL_BUILDER.create(cluster, null);
+ return new RelFieldTrimmer(validator, relBuilder);
+ }
+
+ /**
+ * Converts an unvalidated query's parse tree into a relational expression.
+ *
+ * @param query Query to convert
+ * @param needsValidation Whether to validate the query before converting;
+ * <code>false</code> if the query has already been
+ * validated.
+ * @param top Whether the query is top-level, say if its result
+ * will become a JDBC result set; <code>false</code> if
+ * the query will be part of a view.
+ */
+ public RelRoot convertQuery(
+ SqlNode query,
+ final boolean needsValidation,
+ final boolean top) {
+ if (needsValidation) {
+ query = validator.validate(query);
+ }
- /**
- * @return the RelOptCluster in use.
- */
- public RelOptCluster getCluster() {
- return cluster;
+ RelNode result = convertQueryRecursive(query, top, null).rel;
+ if (top) {
+ if (isStream(query)) {
+ result = new LogicalDelta(cluster, result.getTraitSet(), result);
+ }
+ }
+ RelCollation collation = RelCollations.EMPTY;
+ if (!query.isA(SqlKind.DML)) {
+ if (isOrdered(query)) {
+ collation = requiredCollation(result);
+ }
+ }
+ checkConvertedType(query, result);
+
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(
+ RelOptUtil.dumpPlan(
+ "Plan after converting SqlNode to RelNode",
+ result,
+ false,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
- /**
- * Returns the row-expression builder.
- */
- public RexBuilder getRexBuilder() {
- return rexBuilder;
+ final RelDataType validatedRowType = validator.getValidatedNodeType(query);
+ return RelRoot.of(result, validatedRowType, query.getKind())
+ .withCollation(collation);
+ }
+
+ private static boolean isStream(SqlNode query) {
+ return query instanceof SqlSelect
+ && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ }
+
+ public static boolean isOrdered(SqlNode query) {
+ switch (query.getKind()) {
+ case SELECT:
+ return ((SqlSelect) query).getOrderList() != null
+ && ((SqlSelect) query).getOrderList().size() > 0;
+ case WITH:
+ return isOrdered(((SqlWith) query).body);
+ case ORDER_BY:
+ return ((SqlOrderBy) query).orderList.size() > 0;
+ default:
+ return false;
}
+ }
- /**
- * Returns the number of dynamic parameters encountered during translation;
- * this must only be called after {@link #convertQuery}.
- *
- * @return number of dynamic parameters
- */
- public int getDynamicParamCount() {
- return dynamicParamSqlNodes.size();
+ private RelCollation requiredCollation(RelNode r) {
+ if (r instanceof Sort) {
+ return ((Sort) r).collation;
+ }
+ if (r instanceof Project) {
+ return requiredCollation(((Project) r).getInput());
+ }
+ if (r instanceof Delta) {
+ return requiredCollation(((Delta) r).getInput());
+ }
+ throw new AssertionError();
+ }
+
+ /**
+ * Converts a SELECT statement's parse tree into a relational expression.
+ */
+ public RelNode convertSelect(SqlSelect select, boolean top) {
+ final SqlValidatorScope selectScope = validator.getWhereScope(select);
+ final Blackboard bb = createBlackboard(selectScope, null, top);
+ convertSelectImpl(bb, select);
+ return bb.root;
+ }
+
+ /**
+ * Factory method for creating translation workspace.
+ */
+ protected Blackboard createBlackboard(SqlValidatorScope scope,
+ Map<String, RexNode> nameToNodeMap, boolean top) {
+ return new Blackboard(scope, nameToNodeMap, top);
+ }
+
+ /**
+ * Implementation of {@link #convertSelect(SqlSelect, boolean)};
+ * derived class may override.
+ */
+ protected void convertSelectImpl(
+ final Blackboard bb,
+ SqlSelect select) {
+ convertFrom(
+ bb,
+ select.getFrom());
+ convertWhere(
+ bb,
+ select.getWhere());
+
+ final List<SqlNode> orderExprList = new ArrayList<>();
+ final List<RelFieldCollation> collationList = new ArrayList<>();
+ gatherOrderExprs(
+ bb,
+ select,
+ select.getOrderList(),
+ orderExprList,
+ collationList);
+ final RelCollation collation =
+ cluster.traitSet().canonize(RelCollations.of(collationList));
+
+ if (validator.isAggregate(select)) {
+ convertAgg(
+ bb,
+ select,
+ orderExprList);
+ } else {
+ convertSelectList(
+ bb,
+ select,
+ orderExprList);
}
- /**
- * Returns the type inferred for a dynamic parameter.
- *
- * @param index 0-based index of dynamic parameter
- * @return inferred type, never null
- */
- public RelDataType getDynamicParamType(int index) {
- SqlNode sqlNode = dynamicParamSqlNodes.get(index);
- if (sqlNode == null) {
- throw Util.needToImplement("dynamic param type inference");
- }
- return validator.getValidatedNodeType(sqlNode);
+ if (select.isDistinct()) {
+ distinctify(bb, true);
+ }
+ convertOrder(
+ select, bb, collation, orderExprList, select.getOffset(),
+ select.getFetch());
+ bb.setRoot(bb.root, true);
+ }
+
+ /**
+ * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
+ * a relational expression to make the results unique.
+ *
+ * <p>If the SELECT clause contains duplicate expressions, adds
+ * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are
+ * grouping on the minimal set of keys. The performance gain isn't huge, but
+ * it is difficult to detect these duplicate expressions later.
+ *
+ * @param bb Blackboard
+ * @param checkForDupExprs Check for duplicate expressions
+ */
+ private void distinctify(
+ Blackboard bb,
+ boolean checkForDupExprs) {
+ // Look for duplicate expressions in the project.
+ // Say we have 'select x, y, x, z'.
+ // Then dups will be {[2, 0]}
+ // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
+ RelNode rel = bb.root;
+ if (checkForDupExprs && (rel instanceof LogicalProject)) {
+ LogicalProject project = (LogicalProject) rel;
+ final List<RexNode> projectExprs = project.getProjects();
+ final List<Integer> origins = new ArrayList<>();
+ int dupCount = 0;
+ for (int i = 0; i < projectExprs.size(); i++) {
+ int x = findExpr(projectExprs.get(i), projectExprs, i);
+ if (x >= 0) {
+ origins.add(x);
+ ++dupCount;
+ } else {
+ origins.add(i);
+ }
+ }
+ if (dupCount == 0) {
+ distinctify(bb, false);
+ return;
+ }
+
+ final Map<Integer, Integer> squished = Maps.newHashMap();
+ final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
+ final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ if (origins.get(i) == i) {
+ squished.put(i, newProjects.size());
+ newProjects.add(RexInputRef.of2(i, fields));
+ }
+ }
+ rel =
+ LogicalProject.create(rel, Pair.left(newProjects),
+ Pair.right(newProjects));
+ bb.root = rel;
+ distinctify(bb, false);
+ rel = bb.root;
+
+ // Create the expressions to reverse the mapping.
+ // Project($0, $1, $0, $2).
+ final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ final int origin = origins.get(i);
+ RelDataTypeField field = fields.get(i);
+ undoProjects.add(
+ Pair.of(
+ (RexNode) new RexInputRef(
+ squished.get(origin), field.getType()),
+ field.getName()));
+ }
+
+ rel =
+ LogicalProject.create(rel, Pair.left(undoProjects),
+ Pair.right(undoProjects));
+ bb.setRoot(
+ rel,
+ false);
+
+ return;
}
- /**
- * Returns the current count of the number of dynamic parameters in an
- * EXPLAIN PLAN statement.
- *
- * @param increment if true, increment the count
- * @return the current count before the optional increment
- */
- public int getDynamicParamCountInExplain(boolean increment) {
- int retVal = nDynamicParamsInExplain;
- if (increment) {
- ++nDynamicParamsInExplain;
- }
- return retVal;
+ // Usual case: all of the expressions in the SELECT clause are
+ // different.
+ final ImmutableBitSet groupSet =
+ ImmutableBitSet.range(rel.getRowType().getFieldCount());
+ rel =
+ createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
+ ImmutableList.<AggregateCall>of());
+
+ bb.setRoot(
+ rel,
+ false);
+ }
+
+ private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
+ for (int i = 0; i < count; i++) {
+ RexNode expr = exprs.get(i);
+ if (expr.toString().equals(seek.toString())) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Converts a query's ORDER BY clause, if any.
+ *
+ * @param select Query
+ * @param bb Blackboard
+ * @param collation Collation list
+ * @param orderExprList Method populates this list with orderBy expressions
+ * not present in selectList
+ * @param offset Expression for number of rows to discard before
+ * returning first row
+ * @param fetch Expression for number of rows to fetch
+ */
+ protected void convertOrder(
+ SqlSelect select,
+ Blackboard bb,
+ RelCollation collation,
+ List<SqlNode> orderExprList,
+ SqlNode offset,
+ SqlNode fetch) {
+ if (select.getOrderList() == null
+ || select.getOrderList().getList().isEmpty()) {
+ assert collation.getFieldCollations().isEmpty();
+ if ((offset == null
+ || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
+ && fetch == null) {
+ return;
+ }
}
- /**
- * @return mapping of non-correlated subqueries that have been converted to
- * the constants that they evaluate to
- */
- public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
- return mapConvertedNonCorrSubqs;
+ // Create a sorter using the previously constructed collations.
+ bb.setRoot(
+ LogicalSort.create(bb.root, collation,
+ offset == null ? null : convertExpression(offset),
+ fetch == null ? null : convertExpression(fetch)),
+ false);
+
+ // If extra expressions were added to the project list for sorting,
+ // add another project to remove them. But make the collation empty, because
+ // we can't represent the real collation.
+ //
+ // If it is the top node, use the real collation, but don't trim fields.
+ if (orderExprList.size() > 0 && !bb.top) {
+ final List<RexNode> exprs = new ArrayList<>();
+ final RelDataType rowType = bb.root.getRowType();
+ final int fieldCount =
+ rowType.getFieldCount() - orderExprList.size();
+ for (int i = 0; i < fieldCount; i++) {
+ exprs.add(rexBuilder.makeInputRef(bb.root, i));
+ }
+ bb.setRoot(
+ LogicalProject.create(bb.root, exprs,
+ rowType.getFieldNames().subList(0, fieldCount)),
+ false);
+ }
+ }
+
+ /**
+ * Returns whether a given node contains a {@link SqlInOperator}.
+ *
+ * @param node a RexNode tree
+ */
+ private static boolean containsInOperator(
+ SqlNode node) {
+ try {
+ SqlVisitor<Void> visitor =
+ new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visit(call);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
}
+ }
+
+ /**
+ * Push down all the NOT logical operators into any IN/NOT IN operators.
+ *
+ * @param sqlNode the root node from which to look for NOT operators
+ * @return the transformed SqlNode representation with NOT pushed down.
+ */
+ private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+ SqlCall sqlCall = (SqlCall) sqlNode;
+ if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
+ || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+ for (int i = 0; i < sqlOperands.length; i++) {
+ sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ }
+ return sqlNode;
+ } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode childNode = sqlCall.operand(0);
+ assert childNode instanceof SqlCall;
+ SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+ if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+ SqlNode[] andOperands = childSqlCall.getOperands();
+ SqlNode[] orOperands = new SqlNode[andOperands.length];
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ andOperands[i]);
+ }
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = pushDownNotForIn(orOperands[i]);
+ }
+ return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
+ orOperands[0], orOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+ SqlNode[] orOperands = childSqlCall.getOperands();
+ SqlNode[] andOperands = new SqlNode[orOperands.length];
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ orOperands[i]);
+ }
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = pushDownNotForIn(andOperands[i]);
+ }
+ return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
+ andOperands[0], andOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode[] notOperands = childSqlCall.getOperands();
+ assert notOperands.length == 1;
+ return pushDownNotForIn(notOperands[0]);
+ } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+ SqlNode[] inOperands = childSqlCall.getOperands();
+ SqlInOperator inOp =
+ (SqlInOperator) childSqlCall.getOperator();
+ if (inOp.isNotIn()) {
+ return SqlStdOperatorTable.IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ } else {
+ return SqlStdOperatorTable.NOT_IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ }
+ } else {
+ // childSqlCall is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // sqlNode is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // tree rooted at sqlNode does not contain inOperator
+ return sqlNode;
+ }
+ }
+
+ /**
+ * Converts a WHERE clause.
+ *
+ * @param bb Blackboard
+ * @param where WHERE clause, may be null
+ */
+ private void convertWhere(
+ final Blackboard bb,
+ final SqlNode where) {
+ if (where == null) {
+ return;
+ }
+ SqlNode newWhere = pushDownNotForIn(where);
+ replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ final RexNode convertedWhere = bb.convertExpression(newWhere);
- /**
- * Adds to the current map of non-correlated converted subqueries the
- * elements from another map that contains non-correlated subqueries that
- * have been converted by another SqlToRelConverter.
- *
- * @param alreadyConvertedNonCorrSubqs the other map
- */
- public void addConvertedNonCorrSubqs(Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
- mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
+ // only allocate filter if the condition is not TRUE
+ if (convertedWhere.isAlwaysTrue()) {
+ return;
}
- /**
- * Set a new DefaultValueFactory. To have any effect, this must be called
- * before any convert method.
- *
- * @param factory new DefaultValueFactory
- */
- public void setDefaultValueFactory(DefaultValueFactory factory) {
- defaultValueFactory = factory;
+ final RelNode filter = RelOptUtil.createFilter(bb.root, convertedWhere);
+ final RelNode r;
+ final CorrelationUse p = getCorrelationUse(bb, filter);
+ if (p != null) {
+ assert p.r instanceof Filter;
+ Filter f = (Filter) p.r;
+ r = LogicalFilter.create(f.getInput(), f.getCondition(),
+ ImmutableSet.of(p.id));
+ } else {
+ r = filter;
}
- /**
- * Sets a new SubqueryConverter. To have any effect, this must be called
- * before any convert method.
- *
- * @param converter new SubqueryConverter
- */
- public void setSubqueryConverter(SubqueryConverter converter) {
- subqueryConverter = converter;
+ bb.setRoot(r, false);
+ }
+
+ private void replaceSubqueries(
+ final Blackboard bb,
+ final SqlNode expr,
+ RelOptUtil.Logic logic) {
+ findSubqueries(bb, expr, logic, false);
+ for (SubQuery node : bb.subqueryList) {
+ substituteSubquery(bb, node);
}
+ }
- /**
- * Indicates that the current statement is part of an EXPLAIN PLAN statement
- *
- * @param nDynamicParams number of dynamic parameters in the statement
- */
- public void setIsExplain(int nDynamicParams) {
- isExplain = true;
- nDynamicParamsInExplain = nDynamicParams;
+ private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
+ final RexNode expr = subQuery.expr;
+ if (expr != null) {
+ // Already done.
+ return;
}
- /**
- * Controls whether table access references are converted to physical rels
- * immediately. The optimizer doesn't like leaf rels to have
- * {@link Convention#NONE}. However, if we are doing further conversion
- * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
- * defer conversion. To have any effect, this must be called before any
- * convert method.
- *
- * @param enabled true for immediate conversion (the default); false to
- * generate logical LogicalTableScan instances
- */
- public void enableTableAccessConversion(boolean enabled) {
- shouldConvertTableAccess = enabled;
+ final SqlBasicCall call;
+ final RelNode rel;
+ final SqlNode query;
+ final Pair<RelNode, Boolean> converted;
+ switch (subQuery.node.getKind()) {
+ case CURSOR:
+ convertCursor(bb, subQuery);
+ return;
+
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ case ARRAY_QUERY_CONSTRUCTOR:
+ rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
+ subQuery.expr = bb.register(rel, JoinRelType.INNER);
+ return;
+
+ case IN:
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(1);
+ if (!expand && !(query instanceof SqlNodeList)) {
+ return;
+ }
+ final SqlNode leftKeyNode = call.operand(0);
+
+ final List<RexNode> leftKeys;
+ switch (leftKeyNode.getKind()) {
+ case ROW:
+ leftKeys = Lists.newArrayList();
+ for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
+ leftKeys.add(bb.convertExpression(sqlExpr));
+ }
+ break;
+ default:
+ leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
+ }
+
+ final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
+ if (query instanceof SqlNodeList) {
+ SqlNodeList valueList = (SqlNodeList) query;
+ if (!containsNullLiteral(valueList)
+ && valueList.size() < getInSubqueryThreshold()) {
+ // We're under the threshold, so convert to OR.
+ subQuery.expr =
+ convertInToOr(
+ bb,
+ leftKeys,
+ valueList,
+ isNotIn);
+ return;
+ }
+
+ // Otherwise, let convertExists translate
+ // values list into an inline table for the
+ // reference to Q below.
+ }
+
+ // Project out the search columns from the left side
+
+ // Q1:
+ // "select from emp where emp.deptno in (select col1 from T)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp inner join (select distinct col1 from T)) q
+ // on emp.deptno = q.col1
+ //
+ // Q2:
+ // "select from emp where emp.deptno not in (Q)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp left outer join (select distinct col1, TRUE from T) q
+ // on emp.deptno = q.col1
+ // where emp.deptno <> null
+ // and q.indicator <> TRUE"
+ //
+ final boolean outerJoin = bb.subqueryNeedsOuterJoin
+ || isNotIn
+ || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
+ final RelDataType targetRowType =
+ SqlTypeUtil.promoteToRowType(typeFactory,
+ validator.getValidatedNodeType(leftKeyNode), null);
+ converted =
+ convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic,
+ outerJoin, targetRowType);
+ if (converted.right) {
+ // Generate
+ // emp CROSS JOIN (SELECT COUNT(*) AS c,
+ // COUNT(deptno) AS ck FROM dept)
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RelNode seek = converted.left.getInput(0); // fragile
+ final int keyCount = leftKeys.size();
+ final List<Integer> args = ImmutableIntList.range(0, keyCount);
+ LogicalAggregate aggregate =
+ LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
+ ImmutableList.of(
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ ImmutableList.<Integer>of(), -1, longType, null),
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ args, -1, longType, null)));
+ LogicalJoin join =
+ LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
+ ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
+ bb.setRoot(join, false);
+ }
+ RexNode rex =
+ bb.register(converted.left,
+ outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
+
+ subQuery.expr = translateIn(subQuery, bb.root, rex);
+ if (isNotIn) {
+ subQuery.expr =
+ rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ }
+ return;
+
+ case EXISTS:
+ // "select from emp where exists (select a from T)"
+ //
+ // is converted to the following if the subquery is correlated:
+ //
+ // "select from emp left outer join (select AGG_TRUE() as indicator
+ // from T group by corr_var) q where q.indicator is true"
+ //
+ // If there is no correlation, the expression is replaced with a
+ // boolean indicating whether the subquery returned 0 or >= 1 row.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(0);
+ if (!expand) {
+ return;
+ }
+ converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS,
+ subQuery.logic, true, null);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
+ return;
+ }
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ case SCALAR_QUERY:
+ // Convert the subquery. If it's non-correlated, convert it
+ // to a constant expression.
+ if (!expand) {
+ return;
+ }
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(0);
+ converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR,
+ subQuery.logic, true, null);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
+ return;
+ }
+ rel = convertToSingleValueSubq(query, converted.left);
+ subQuery.expr = bb.register(rel, JoinRelType.LEFT);
+ return;
+
+ case SELECT:
+ // This is used when converting multiset queries:
+ //
+ // select * from unnest(select multiset[deptno] from emps);
+ //
+ converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR,
+ subQuery.logic, true, null);
+ assert !converted.right;
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ default:
+ throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
+ }
+ }
+
+ private RexNode translateIn(SubQuery subQuery, RelNode root,
+ final RexNode rex) {
+ switch (subQuery.logic) {
+ case TRUE:
+ return rexBuilder.makeLiteral(true);
+
+ case UNKNOWN_AS_FALSE:
+ assert rex instanceof RexRangeRef;
+ final int fieldCount = rex.getType().getFieldCount();
+ RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
+
+ // Then append the IS NOT NULL(leftKeysForIn).
+ //
+ // RexRangeRef contains the following fields:
+ // leftKeysForIn,
+ // rightKeysForIn (the original subquery select list),
+ // nullIndicator
+ //
+ // The first two lists contain the same number of fields.
+ final int k = (fieldCount - 1) / 2;
+ for (int i = 0; i < k; i++) {
+ rexNode =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.AND,
+ rexNode,
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.IS_NOT_NULL,
+ rexBuilder.makeFieldAccess(rex, i)));
+ }
+ return rexNode;
+
+ case TRUE_FALSE_UNKNOWN:
+ case UNKNOWN_AS_TRUE:
+ // select e.deptno,
+ // case
+ // when ct.c = 0 then false
+ // when dt.i is not null then true
+ // when e.deptno is null then null
+ // when ct.ck < ct.c then null
+ // else false
+ // end
+ // from e
+ // cross join (select count(*) as c, count(deptno) as ck from v) as ct
+ // left join (select distinct deptno, true as i from v) as dt
+ // on e.deptno = dt.deptno
+ final Join join = (Join) root;
+ final Project left = (Project) join.getLeft();
+ final RelNode leftLeft = ((Join) left.getInput()).getLeft();
+ final int leftLeftCount = leftLeft.getRowType().getFieldCount();
+ final RelDataType nullableBooleanType =
+ typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
+ final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
+ final RexNode iRef =
+ rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+
+ final RexLiteral zero =
+ rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
+ final RexNode unknownLiteral =
+ rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
+
+ final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
+ falseLiteral,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
+ trueLiteral);
+ final JoinInfo joinInfo = join.analyzeCondition();
+ for (int leftKey : joinInfo.leftKeys) {
+ final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
+ unknownLiteral);
+ }
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
+ unknownLiteral,
+ falseLiteral);
+
+ return rexBuilder.makeCall(
+ nullableBooleanType,
+ SqlStdOperatorTable.CASE,
+ args.build());
+
+ default:
+ throw new AssertionError(subQuery.logic);
}
+ }
- /**
- * Controls whether instances of
- * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
- * may not be supported by all physical implementations. To have any effect,
- * this must be called before any convert method.
- *
- * @param enabled true to allow LogicalValues to be generated (the default);
- * false to force substitution of Project+OneRow instead
- */
- public void enableValuesRelCreation(boolean enabled) {
- shouldCreateValuesRel = enabled;
- }
-
- private void checkConvertedType(SqlNode query, RelNode result) {
- if (!query.isA(SqlKind.DML)) {
- // Verify that conversion from SQL to relational algebra did
- // not perturb any type information. (We can't do this if the
- // SQL statement is something like an INSERT which has no
- // validator type information associated with its result,
- // hence the namespace check above.)
- RelDataType convertedRowType = result.getRowType();
- if (!checkConvertedRowType(query, convertedRowType)) {
- RelDataType validatedRowType = validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
- throw Util.newInternal("Conversion to relational algebra failed to " + "preserve datatypes:\n" + "validated type:\n" + validatedRowType.getFullTypeString() + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n" + RelOptUtil.toString(result));
- }
+ private static boolean containsNullLiteral(SqlNodeList valueList) {
+ for (SqlNode node : valueList.getList()) {
+ if (node instanceof SqlLiteral) {
+ SqlLiteral lit = (SqlLiteral) node;
+ if (lit.getValue() == null) {
+ return true;
}
+ }
}
-
- public RelNode flattenTypes(RelNode rootRel, boolean restructure) {
- RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
- return typeFlattener.rewrite(rootRel, restructure);
+ return false;
+ }
+
+ /**
+ * Determines if a subquery is non-correlated and if so, converts it to a
+ * constant.
+ *
+ * @param subQuery the call that references the subquery
+ * @param bb blackboard used to convert the subquery
+ * @param converted RelNode tree corresponding to the subquery
+ * @param isExists true if the subquery is part of an EXISTS expression
+ * @return if the subquery can be converted to a constant
+ */
+ private boolean convertNonCorrelatedSubQuery(
+ SubQuery subQuery,
+ Blackboard bb,
+ RelNode converted,
+ boolean isExists) {
+ SqlCall call = (SqlBasicCall) subQuery.node;
+ if (subqueryConverter.canConvertSubquery()
+ && isSubQueryNonCorrelated(converted, bb)) {
+ // First check if the subquery has already been converted
+ // because it's a nested subquery. If so, don't re-evaluate
+ // it again.
+ RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
+ if (constExpr == null) {
+ constExpr =
+ subqueryConverter.convertSubquery(
+ call,
+ this,
+ isExists,
+ isExplain);
+ }
+ if (constExpr != null) {
+ subQuery.expr = constExpr;
+ mapConvertedNonCorrSubqs.put(call, constExpr);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Converts the RelNode tree for a select statement to a select that
+ * produces a single value.
+ *
+ * @param query the query
+ * @param plan the original RelNode tree corresponding to the statement
+ * @return the converted RelNode tree
+ */
+ public RelNode convertToSingleValueSubq(
+ SqlNode query,
+ RelNode plan) {
+ // Check whether query is guaranteed to produce a single value.
+ if (query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) query;
+ SqlNodeList selectList = select.getSelectList();
+ SqlNodeList groupList = select.getGroup();
+
+ if ((selectList.size() == 1)
+ && ((groupList == null) || (groupList.size() == 0))) {
+ SqlNode selectExpr = selectList.get(0);
+ if (selectExpr instanceof SqlCall) {
+ SqlCall selectExprCall = (SqlCall) selectExpr;
+ if (Util.isSingleValue(selectExprCall)) {
+ return plan;
+ }
+ }
+
+ // If there is a limit with 0 or 1,
+ // it is ensured to produce a single value
+ if (select.getFetch() != null
+ && select.getFetch() instanceof SqlNumericLiteral) {
+ SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
+ if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
+ return plan;
+ }
+ }
+ }
+ } else if (query instanceof SqlCall) {
+ // If the query is (values ...),
+ // it is necessary to look into the operands to determine
+ // whether SingleValueAgg is necessary
+ SqlCall exprCall = (SqlCall) query;
+ if (exprCall.getOperator()
+ instanceof SqlValuesOperator
+ && Util.isSingleValue(exprCall)) {
+ return plan;
+ }
}
- /**
- * If subquery is correlated and decorrelation is enabled, performs
- * decorrelation.
- *
- * @param query Query
- * @param rootRel Root relational expression
- * @return New root relational expression after decorrelation
- */
- public RelNode decorrelate(SqlNode query, RelNode rootRel) {
- if (!enableDecorrelation()) {
- return rootRel;
- }
- final RelNode result = decorrelateQuery(rootRel);
- if (result != rootRel) {
- checkConvertedType(query, result);
- }
- return result;
+ // If not, project SingleValueAgg
+ return RelOptUtil.createSingleValueAggRel(
+ cluster,
+ plan);
+ }
+
+ /**
+ * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
+ *
+ * @param leftKeys LHS
+ * @param valuesList RHS
+ * @param isNotIn is this a NOT IN operator
+ * @return converted expression
+ */
+ private RexNode convertInToOr(
+ final Blackboard bb,
+ final List<RexNode> leftKeys,
+ SqlNodeList valuesList,
+ boolean isNotIn) {
+ final List<RexNode> comparisons = new ArrayList<>();
+ for (SqlNode rightVals : valuesList) {
+ RexNode rexComparison;
+ if (leftKeys.size() == 1) {
+ rexComparison =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.EQUALS,
+ leftKeys.get(0),
+ rexBuilder.ensureType(leftKeys.get(0).getType(),
+ bb.convertExpression(rightVals), true));
+ } else {
+ assert rightVals instanceof SqlCall;
+ final SqlBasicCall call = (SqlBasicCall) rightVals;
+ assert (call.getOperator() instanceof SqlRowOperator)
+ && call.operandCount() == leftKeys.size();
+ rexComparison =
+ RexUtil.composeConjunction(
+ rexBuilder,
+ Iterables.transform(
+ Pair.zip(leftKeys, call.getOperandList()),
+ new Function<Pair<RexNode, SqlNode>, RexNode>() {
+ public RexNode apply(Pair<RexNode, SqlNode> pair) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ pair.left,
+ rexBuilder.ensureType(pair.left.getType(),
+ bb.convertExpression(pair.right), true));
+ }
+ }),
+ false);
+ }
+ comparisons.add(rexComparison);
}
- /**
- * Walks over a tree of relational expressions, replacing each
- * {@link RelNode} with a 'slimmed down' relational expression that projects
- * only the fields required by its consumer.
- *
- * <p>This may make things easier for the optimizer, by removing crud that
- * would expand the search space, but is difficult for the optimizer itself
- * to do it, because optimizer rules must preserve the number and type of
- * fields. Hence, this transform that operates on the entire tree, similar
- * to the {@link RelStructuredTypeFlattener type-flattening transform}.
- *
- * <p>Currently this functionality is disabled in farrago/luciddb; the
- * default implementation of this method does nothing.
- *
- * @param rootRel Relational expression that is at the root of the tree
- * @return Trimmed relational expression
- */
- public RelNode trimUnusedFields(RelNode rootRel) {
- // Trim fields that are not used by their consumer.
- if (isTrimUnusedFields()) {
- final RelFieldTrimmer trimmer = newFieldTrimmer();
- rootRel = trimmer.trim(rootRel);
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
- }
- return rootRel;
+ RexNode result =
+ RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+ assert result != null;
+
+ if (isNotIn) {
+ result =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.NOT,
+ result);
}
- /**
- * Creates a RelFieldTrimmer.
- *
- * @return Field trimmer
- */
- protected RelFieldTrimmer newFieldTrimmer() {
- return new RelFieldTrimmer(validator);
+ return result;
+ }
+
+ /**
+ * Gets the list size threshold under which {@link #convertInToOr} is used.
+ * Lists of this size or greater will instead be converted to use a join
+ * against an inline table
+ * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a
+ * predicate. A threshold of 0 forces usage of an inline table in all cases; a
+ * threshold of Integer.MAX_VALUE forces usage of OR in all cases
+ *
+ * @return threshold, default {@link #IN_SUBQUERY_THRESHOLD}
+ */
+ protected int getInSubqueryThreshold() {
+ /* OVERRIDE POINT */
+ return Integer.MAX_VALUE;
+ }
+
+ /**
+ * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
+ * produces an indicator variable, and the result is a relational expression
+ * which outer joins that indicator to the original query. After performing
+ * the outer join, the condition will be TRUE if the EXISTS condition holds,
+ * NULL otherwise.
+ *
+ * @param seek A query, for example 'select * from emp' or
+ * 'values (1,2,3)' or '('Foo', 34)'.
+ * @param subqueryType Whether sub-query is IN, EXISTS or scalar
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept an
+ * approximation (say representing UNKNOWN as FALSE)
+ * @param needsOuterJoin Whether an outer join is needed
+ * @return join expression
+ * @pre extraExpr == null || extraName != null
+ */
+ private Pair<RelNode, Boolean> convertExists(
+ SqlNode seek,
+ RelOptUtil.SubqueryType subqueryType,
+ RelOptUtil.Logic logic,
+ boolean needsOuterJoin,
+ RelDataType targetDataType) {
+ final SqlValidatorScope seekScope =
+ (seek instanceof SqlSelect)
+ ? validator.getSelectScope((SqlSelect) seek)
+ : null;
+ final Blackboard seekBb = createBlackboard(seekScope, null, false);
+ RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
+
+ return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic,
+ needsOuterJoin);
+ }
+
+ private RelNode convertQueryOrInList(
+ Blackboard bb,
+ SqlNode seek,
+ RelDataType targetRowType) {
+ // NOTE: Once we start accepting single-row queries as row constructors,
+ // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
+ // Z)). The SQL standard resolves the ambiguity by saying that a lone
+ // select should be interpreted as a table expression, not a row
+ // expression. The semantic difference is that a table expression can
+ // return multiple rows.
+ if (seek instanceof SqlNodeList) {
+ return convertRowValues(
+ bb,
+ seek,
+ ((SqlNodeList) seek).getList(),
+ false,
+ targetRowType);
+ } else {
+ return convertQueryRecursive(seek, false, null).project();
+ }
+ }
+
+ private RelNode convertRowValues(
+ Blackboard bb,
+ SqlNode rowList,
+ Collection<SqlNode> rows,
+ boolean allowLiteralsOnly,
+ RelDataType targetRowType) {
+ // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
+ // literals into a single LogicalValues; this gives the optimizer a smaller
+ // input tree. For everything else (computed expressions, row
+ // subqueries), we union each row in as a projection on top of a
+ // LogicalOneRow.
+
+ final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
+ ImmutableList.builder();
+ final RelDataType rowType;
+ if (targetRowType != null) {
+ rowType = targetRowType;
+ } else {
+ rowType =
+ SqlTypeUtil.promoteToRowType(
+ typeFactory,
+ validator.getValidatedNodeType(rowList),
+ null);
}
- /**
- * Converts an unvalidated query's parse tree into a relational expression.
- *
- * @param query Query to convert
- * @param needsValidation Whether to validate the query before converting;
- * <code>false</code> if the query has already been
- * validated.
- * @param top Whether the query is top-level, say if its result
- * will become a JDBC result set; <code>false</code> if
- * the query will be part of a view.
- */
- public RelNode convertQuery(SqlNode query, final boolean needsValidation, final boolean top) {
- if (needsValidation) {
- query = validator.validate(query);
+ final List<RelNode> unionInputs = new ArrayList<>();
+ for (SqlNode node : rows) {
+ SqlBasicCall call;
+ if (isRowConstructor(node)) {
+ call = (SqlBasicCall) node;
+ ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
+ for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ operand.e,
+ bb,
+ rowType,
+ operand.i);
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
+ if ((rexLiteral == null) || !shouldCreateValuesRel) {
+ // fallback to convertRowConstructor
+ tuple = null;
+ break;
+ }
+ tuple.add(rexLiteral);
+ }
+ if (tuple != null) {
+ tupleList.add(tuple.build());
+ continue;
+ }
+ } else {
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ node,
+ bb,
+ rowType,
+ 0);
+ if ((rexLiteral != null) && shouldCreateValuesRel) {
+ tupleList.add(ImmutableList.of(rexLiteral));
+ continue;
+ } else {
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
}
- RelNode result = convertQueryRecursive(query, top, null);
- if (top && isStream(query)) {
- result = new LogicalDelta(cluster, result.getTraitSet(), result);
- }
- checkConvertedType(query, result);
+ // convert "1" to "row(1)"
+ call =
+ (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
+ SqlParserPos.ZERO,
+ node);
+ }
+ unionInputs.add(convertRowConstructor(bb, call));
+ }
+ LogicalValues values =
+ LogicalValues.create(cluster, rowType, tupleList.build());
+ RelNode resultRel;
+ if (unionInputs.isEmpty()) {
+ resultRel = values;
+ } else {
+ if (!values.getTuples().isEmpty()) {
+ unionInputs.add(values);
+ }
+ resultRel = LogicalUnion.create(unionInputs, true);
+ }
+ leaves.add(resultRel);
+ return resultRel;
+ }
+
+ private RexLiteral convertLiteralInValuesList(
+ SqlNode sqlNode,
+ Blackboard bb,
+ RelDataType rowType,
+ int iField) {
+ if (!(sqlNode instanceof SqlLiteral)) {
+ return null;
+ }
+ RelDataTypeField field = rowType.getFieldList().get(iField);
+ RelDataType type = field.getType();
+ if (type.isStruct()) {
+ // null literals for weird stuff like UDT's need
+ // special handling during type flattening, so
+ // don't use LogicalValues for those
+ return null;
+ }
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode", result, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
+ RexNode literalExpr =
+ exprConverter.convertLiteral(
+ bb,
+ (SqlLiteral) sqlNode);
+
+ if (!(literalExpr instanceof RexLiteral)) {
+ assert literalExpr.isA(SqlKind.CAST);
+ RexNode child = ((RexCall) literalExpr).getOperands().get(0);
+ assert RexLiteral.isNullLiteral(child);
- return result;
+ // NOTE jvs 22-Nov-2006: we preserve type info
+ // in LogicalValues digest, so it's OK to lose it here
+ return (RexLiteral) child;
}
- private static boolean isStream(SqlNode query) {
- return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ RexLiteral literal = (RexLiteral) literalExpr;
+
+ Comparable value = literal.getValue();
+
+ if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
+ BigDecimal roundedValue =
+ NumberUtil.rescaleBigDecimal(
+ (BigDecimal) value,
+ type.getScale());
+ return rexBuilder.makeExactLiteral(
+ roundedValue,
+ type);
}
- protected boolean checkConvertedRowType(SqlNode query, RelDataType convertedRowType) {
- RelDataType validatedRowType = validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
+ if ((value instanceof NlsString)
+ && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+ // pad fixed character type
+ NlsString unpadded = (NlsString) value;
+ return rexBuilder.makeCharLiteral(
+ new NlsString(
+ Spaces.padRight(unpadded.getValue(), type.getPrecision()),
+ unpadded.getCharsetName(),
+ unpadded.getCollation()));
+ }
+ return literal;
+ }
- return RelOptUtil.equal("validated row type", validatedRowType, "converted row type", convertedRowType, false);
+ private boolean isRowConstructor(SqlNode node) {
+ if (!(node.getKind() == SqlKind.ROW)) {
+ return false;
+ }
+ SqlCall call = (SqlCall) node;
+ return call.getOperator().getName().equalsIgnoreCase("row");
+ }
+
+ /**
+ * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
+ * inside SQL parse tree. Does not traverse inside queries.
+ *
+ * @param bb blackboard
+ * @param node the SQL parse tree
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept
+ * an approximation (say representing UNKNOWN as FALSE)
+ * @param registerOnlyScalarSubqueries if set to true and the parse tree
+ * corresponds to a variation of a select
+ * node, only register it if it's a scalar
+ * subquery
+ */
+ private void findSubqueries(
+ Blackboard bb,
+ SqlNode node,
+ RelOptUtil.Logic logic,
+ boolean registerOnlyScalarSubqueries) {
+ final SqlKind kind = node.getKind();
+ switch (kind) {
+ case EXISTS:
+ case SELECT:
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ case ARRAY_QUERY_CONSTRUCTOR:
+ case CURSOR:
+ case SCALAR_QUERY:
+ if (!registerOnlyScalarSubqueries
+ || (kind == SqlKind.SCALAR_QUERY)) {
+ bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
+ }
+ return;
+ case IN:
+ if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
+ logic = logic.negate();
+ }
+ break;
+ case NOT:
+ logic = logic.negate();
+ break;
+ }
+ if (node instanceof SqlCall) {
+ if (kind == SqlKind.OR
+ || kind == SqlKind.NOT) {
+ // It's always correct to outer join subquery with
+ // containing query; however, when predicates involve Or
+ // or NOT, outer join might be necessary.
+ bb.subqueryNeedsOuterJoin = true;
+ }
+ for (SqlNode operand : ((SqlCall) node).getOperandList()) {
+ if (operand != null) {
+ // In the case of an IN expression, locate scalar
+ // subqueries so we can convert them to constants
+ findSubqueries(
+ bb,
+ operand,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
+ }
+ } else if (node instanceof SqlNodeList) {
+ for (SqlNode child : (SqlNodeList) node) {
+ findSubqueries(
+ bb,
+ child,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
}
- protected RelDataType uniquifyFields(RelDataType rowType) {
- return validator.getTypeFactory().createStructType(RelOptUtil.getFieldTypeList(rowType), SqlValidatorUtil.uniquify(rowType.getFieldNames()));
+ // Now that we've located any scalar subqueries inside the IN
+ // expression, register the IN expression itself. We need to
+ // register the scalar subqueries first so they can be converted
+ // before the IN expression is converted.
+ if (kind == SqlKind.IN) {
+ if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
+ && !validator.getValidatedNodeType(node).isNullable()) {
+ logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
+ }
+ // TODO: This conversion is only valid in the WHERE clause
+ if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE
+ && !bb.subqueryNeedsOuterJoin) {
+ logic = RelOptUtil.Logic.TRUE;
+ }
+ bb.registerSubquery(node, logic);
+ }
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format.
+ *
+ * @param node Expression to translate
+ * @return Converted expression
+ */
+ public RexNode convertExpression(
+ SqlNode node) {
+ Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
+ final ParameterScope scope =
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+ final Blackboard bb = createBlackboard(scope, null, false);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format,
+ * mapping identifier references to predefined expressions.
+ *
+ * @param node Expression to translate
+ * @param nameToNodeMap map from String to {@link RexNode}; when an
+ * {@link SqlIdentifier} is encountered, it is used as a
+ * key and translated to the corresponding value from
+ * this map
+ * @return Converted expression
+ */
+ public RexNode convertExpression(
+ SqlNode node,
+ Map<String, RexNode> nameToNodeMap) {
+ final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
+ for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
+ nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
+ }
+ final ParameterScope scope =
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+ final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts a non-standard expression.
+ *
+ * <p>This method is an extension-point that derived classes can override. If
+ * this method returns a null result, the normal expression translation
+ * process will proceed. The default implementation always returns null.
+ *
+ * @param node Expression
+ * @param bb Blackboard
+ * @return null to proceed with the usual expression translation process
+ */
+ protected RexNode convertExtendedExpression(
+ SqlNode node,
+ Blackboard bb) {
+ return null;
+ }
+
+ private RexNode convertOver(Blackboard bb, SqlNode node) {
+ SqlCall call = (SqlCall) node;
+ SqlCall aggCall = call.operand(0);
+ SqlNode windowOrRef = call.operand(1);
+ final SqlWindow window =
+ validator.resolveWindow(windowOrRef, bb.scope, true);
+ // ROW_NUMBER() expects specific kind of framing.
+ if (aggCall.getOperator() == SqlStdOperatorTable.ROW_NUMBER) {
+ window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO));
+ window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO));
+ window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO));
+ }
+ final SqlNodeList partitionList = window.getPartitionList();
+ final ImmutableList.Builder<RexNode> partitionKeys =
+ ImmutableList.builder();
+ for (SqlNode partition : partitionList) {
+ partitionKeys.add(bb.convertExpression(partition));
+ }
+ RexNode lowerBound = bb.convertExpression(window.getLowerBound());
+ RexNode upperBound = bb.convertExpression(window.getUpperBound());
+ SqlNodeList orderList = window.getOrderList();
+ if ((orderList.size() == 0) && !window.isRows()) {
+ // A logical range requires an ORDER BY clause. Use the implicit
+ // ordering of this relation. There must be one, otherwise it would
+ // have failed validation.
+ orderList = bb.scope.getOrderList();
+ if (orderList == null) {
+ throw new AssertionError(
+ "Relation should have sort key for implicit ORDER BY");
+ }
+ }
+ final ImmutableList.Builder<RexFieldCollation> orderKeys =
+ ImmutableList.builder();
+ final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
+ for (SqlNode order : orderList) {
+ flags.clear();
+ RexNode e = bb.convertSortExpression(order, flags);
+ orderKeys.add(new RexFieldCollation(e, flags));
+ }
+ try {
+ Util.permAssert(bb.window == null, "already in window agg mode");
+ bb.window = window;
+ RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
+ rexAgg =
+ rexBuilder.ensureType(
+ validator.getValidatedNodeType(call), rexAgg, false);
+
+ // Walk over the tree and apply 'over' to all agg functions. This is
+ // necessary because the returned expression is not necessarily a call
+ // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
+ final RexShuttle visitor =
+ new HistogramShuttle(
+ partitionKeys.build(), orderKeys.build(),
+ RexWindowBound.create(window.getLowerBound(), lowerBound),
+ RexWindowBound.create(window.getUpperBound(), upperBound),
+ window);
+ return rexAgg.accept(visitor);
+ } finally {
+ bb.window = null;
+ }
+ }
+
+ /**
+ * Converts a FROM clause into a relational expression.
+ *
+ * @param bb Scope within which to resolve identifiers
+ * @param from FROM clause of a query. Examples include:
+ *
+ * <ul>
+ * <li>a single table ("SALES.EMP"),
+ * <li>an aliased table ("EMP AS E"),
+ * <li>a list of tables ("EMP, DEPT"),
+ * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
+ * DEPT.DEPTNO"),
+ * <li>a VALUES clause ("VALUES ('Fred', 20)"),
+ * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
+ * <li>or any combination of the above.
+ * </ul>
+ */
+ protected void convertFrom(
+ Blackboard bb,
+ SqlNode from) {
+ final SqlCall call;
+ final SqlNode[] operands;
+ switch (from.getKind()) {
+ case AS:
+ convertFrom(bb, ((SqlCall) from).operand(0));
+ return;
+
+ case WITH_ITEM:
+ convertFrom(bb, ((SqlWithItem) from).query);
+ return;
+
+ case WITH:
+ convertFrom(bb, ((SqlWith) from).body);
+ return;
+
+ case TABLESAMPLE:
+ operands = ((SqlBasicCall) from).getOperands();
+ SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
+ if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
+ String sampleName =
+ ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
+ .getName();
+ datasetStack.push(sampleName);
+ convertFrom(bb, operands[0]);
+ datasetStack.pop();
+ } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
+ SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
+ (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
+ convertFrom(bb, operands[0]);
+ RelOptSamplingParameters params =
+ new RelOptSamplingParameters(
+ tableSampleSpec.isBernoulli(),
+ tableSampleSpec.getSamplePercentage(),
+ tableSampleSpec.isRepeatable(),
+ tableSampleSpec.getRepeatableSeed());
+ bb.setRoot(new Sample(cluster, bb.root, params), false);
+ } else {
+ throw Util.newInternal(
+ "unknown TABLESAMPLE type: " + sampleSpec);
+ }
+ return;
+
+ case IDENTIFIER:
+ final SqlValidatorNamespace fromNamespace =
+ validator.getNamespace(from).resolve();
+ if (fromNamespace.getNode() != null) {
+ convertFrom(bb, fromNamespace.getNode());
+ return;
+ }
+ final String datasetName =
+ datasetStack.isEmpty() ? null : datasetStack.peek();
+ boolean[] usedDataset = {false};
+ RelOptTable table =
+ SqlValidatorUtil.getRelOptTable(
+ fromNamespace,
+ catalogReader,
+ datasetName,
+ usedDataset);
+ final RelNode tableRel;
+ if (shouldConvertTableAccess) {
+ tableRel = toRel(table);
+ } else {
+ tableRel = LogicalTableScan.create(cluster, table);
+ }
+ bb.setRoot(tableRel, true);
+ if (usedDataset[0]) {
+ bb.setDataset(datasetName);
+ }
+ return;
+
+ case JOIN:
+ final SqlJoin join = (SqlJoin) from;
+ final SqlValidatorScope scope = validator.getJoinScope(from);
+ final Blackboard fromBlackboard = createBlackboard(scope, null, false);
+ SqlNode left = join.getLeft();
+ SqlNode right = join.getRight();
+ final boolean isNatural = join.isNatural();
+ final JoinType joinType = join.getJoinType();
+ final SqlValidatorScope leftScope =
+ Util.first(validator.getJoinScope(left),
+ ((DelegatingScope) bb.scope).getParent());
+ final Blackboard leftBlackboard =
+ createBlackboard(leftScope, null, false);
+ final SqlValidatorScope rightScope =
+ Util.first(validator.getJoinScope(right),
+ ((DelegatingScope) bb.scope).getParent());
+ final Blackboard rightBlackboard =
+ createBlackboard(rightScope, null, false);
+ convertFrom(leftBlackboard, left);
+ RelNode leftRel = leftBlackboard.root;
+ convertFrom(rightBlackboard, right);
+ RelNode rightRel = rightBlackboard.root;
+ JoinRelType convertedJoinType = convertJoinType(joinType);
+ RexNode conditionExp;
+ final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
+ final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
+ if (isNatural) {
+ final RelDataType leftRowType = leftNamespace.getRowType();
+ final RelDataType rightRowType = rightNamespace.getRowType();
+ final List<String> columnList =
+ SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType,
+ rightRowType);
+ conditionExp = convertUsing(leftNamespace, rightNamespace,
+ columnList);
+ } else {
+ conditionExp =
+ convertJoinCondition(
+ fromBlackboard,
+ leftNamespace,
+ rightNamespace,
+ join.getCondition(),
+ join.getConditionType(),
+ leftRel,
+ rightRel);
+ }
+
+ final RelNode joinRel =
+ createJoin(
+ fromBlackboard,
+ leftRel,
+ rightRel,
+ conditionExp,
+ convertedJoinType);
+ bb.setRoot(joinRel, false);
+ return;
+
+ case SELECT:
+ case INTERSECT:
+ case EXCEPT:
+ case UNION:
+ final RelNode rel = convertQueryRecursive(from, false, null).project();
+ bb.setRoot(rel, true);
+ return;
+
+ case VALUES:
+ convertValuesImpl(bb, (SqlCall) from, null);
+ return;
+
+ case UNNEST:
+ call = (SqlCall) from;
+ final SqlNode node = call.operand(0);
+ final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator();
+ replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ final RelNode childRel =
+ RelOptUtil.createProject(
+ (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
+ Collections.singletonList(bb.convertExpression(node)),
+ Collections.singletonList(validator.deriveAlias(node, 0)),
+ true);
+
+ Uncollect uncollect =
+ new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
+ childRel, operator.withOrdinality);
+ bb.setRoot(uncollect, true);
+ return;
+
+ case COLLECTION_TABLE:
+ call = (SqlCall) from;
+
+ // Dig out real call; TABLE() wrapper is just syntactic.
+ assert call.getOperandList().size() == 1;
+ final SqlCall call2 = call.operand(0);
+ convertCollectionTable(bb, call2);
+ return;
+
+ default:
+ throw Util.newInternal("not a join operator " + from);
+ }
+ }
+
+ protected void convertCollectionTable(
+ Blackboard bb,
+ SqlCall call) {
+ final SqlOperator operator = call.getOperator();
+ if (operator == SqlStdOperatorTable.TABLESAMPLE) {
+ final String sampleName =
+ SqlLiteral.stringValue(call.operand(0));
+ datasetStack.push(sampleName);
+ SqlCall cursorCall = call.operand(1);
+ SqlNode query = cursorCall.operand(0);
+ RelNode converted = convertQuery(query, false, false).rel;
+ bb.setRoot(converted, false);
+ datasetStack.pop();
+ return;
+ }
+ replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // Expand table macro if possible. It's more efficient than
+ // LogicalTableFunctionScan.
+ final SqlCallBinding callBinding =
+ new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
+ if (operator instanceof SqlUserDefinedTableMacro) {
+ final SqlUserDefinedTableMacro udf =
+ (SqlUserDefinedTableMacro) operator;
+ final TranslatableTable table =
+ udf.getTable(typeFactory, callBinding.operands());
+ final RelDataType rowType = table.getRowType(typeFactory);
+ RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
+ RelNode converted = toRel(relOptTable);
+ bb.setRoot(converted, true);
+ return;
}
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertSelect(SqlSelect select) {
- final SqlValidatorScope selectScope = validator.getWhereScope(select);
- final Blackboard bb = createBlackboard(selectScope, null);
- convertSelectImpl(bb, select);
- return bb.root;
+ Type elementType;
+ if
<TRUNCATED>
[10/50] [abbrv] kylin git commit: KYLIN-1038 retry on job failure
Posted by sh...@apache.org.
KYLIN-1038 retry on job failure
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3af7d4a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3af7d4a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3af7d4a7
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 3af7d4a72c8308f00fe95276b08f05709eaa62e5
Parents: 17c33dc
Author: shaofengshi <sh...@apache.org>
Authored: Sun Feb 14 21:17:12 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:13:16 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 3 ++
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../kylin/job/execution/AbstractExecutable.java | 57 +++++++++++++-------
.../job/execution/DefaultChainedExecutable.java | 5 ++
4 files changed, 50 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3af7d4a7/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 78a564d..d694e9f 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -23,6 +23,9 @@ kylin.hbase.cluster.fs=
kylin.job.mapreduce.default.reduce.input.mb=500
+# max job retry on error, default 0: no retry
+kylin.job.retry=0
+
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
# It should not be set to "true" unless you're NOT running Kylin.sh on a hadoop client machine
http://git-wip-us.apache.org/repos/asf/kylin/blob/3af7d4a7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 826a28c..487f78e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -538,6 +538,10 @@ public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("crossdomain.enable", "true"));
}
+ public int getJobRetry() {
+ return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
+ }
+
public String toString() {
return getMetadataUrl();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3af7d4a7/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index e1d7106..8d5fea5 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -18,13 +18,10 @@
package org.apache.kylin.job.execution;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
@@ -35,10 +32,12 @@ import org.apache.kylin.job.manager.ExecutableManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
/**
*/
@@ -50,6 +49,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected static final String END_TIME = "endTime";
protected static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
+ protected int retry = 0;
private String name;
private String id;
@@ -99,15 +99,30 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
logger.info("Executing >>>>>>>>>>>>> " + this.getName() + " <<<<<<<<<<<<<");
Preconditions.checkArgument(executableContext instanceof DefaultContext);
- ExecuteResult result;
- try {
- onExecuteStart(executableContext);
- result = doWork(executableContext);
- } catch (Throwable e) {
- logger.error("error running Executable", e);
- onExecuteError(e, executableContext);
- throw new ExecuteException(e);
+ ExecuteResult result = null;
+
+ onExecuteStart(executableContext);
+ Throwable exception;
+ do {
+ if (retry > 0) {
+ logger.info("Retry " + retry);
+ }
+ exception = null;
+ result = null;
+ try {
+ result = doWork(executableContext);
+ } catch (Throwable e) {
+ logger.error("error running Executable", e);
+ exception = e;
+ }
+ retry++;
+ } while (((result != null && result.succeed() == false) || exception != null) && needRetry() == true);
+
+ if (exception != null) {
+ onExecuteError(exception, executableContext);
+ throw new ExecuteException(exception);
}
+
onExecuteFinished(result, executableContext);
return result;
}
@@ -301,6 +316,10 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
return status == ExecutableState.DISCARDED;
}
+ protected boolean needRetry() {
+ return this.retry <= KylinConfig.getInstanceFromEnv().getJobRetry();
+ }
+
@Override
public String toString() {
return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()).toString();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3af7d4a7/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 2e95711..7403715 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -106,6 +106,11 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
return subTasks;
}
+ @Override
+ protected boolean needRetry() {
+ return false;
+ }
+
public final AbstractExecutable getTaskByName(String name) {
for (AbstractExecutable task : subTasks) {
if (task.getName() != null && task.getName().equalsIgnoreCase(name)) {
[09/50] [abbrv] kylin git commit: KYLIN-1474 expose list,
remove and cat in metastore.sh
Posted by sh...@apache.org.
KYLIN-1474 expose list, remove and cat in metastore.sh
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/17c33dc4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/17c33dc4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/17c33dc4
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 17c33dc4929121cdb48c3904dc337cd456af845d
Parents: 11dc04b
Author: John Zhao <yu...@ebay.com>
Authored: Mon Mar 7 14:05:59 2016 -0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 8 10:31:12 2016 +0800
----------------------------------------------------------------------
build/bin/metastore.sh | 24 ++++++++++++++++++++
.../kylin/common/persistence/ResourceTool.java | 3 +++
2 files changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/17c33dc4/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 05041f9..8908351 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -47,6 +47,27 @@ then
echo "Starting restoring $_file"
${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool upload $_file
+elif [ "$1" == "list" ]
+then
+
+ _file=$2
+ echo "Starting list $_file"
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool list $_file
+
+elif [ "$1" == "remove" ]
+then
+
+ _file=$2
+ echo "Starting remove $_file"
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool remove $_file
+
+elif [ "$1" == "cat" ]
+then
+
+ _file=$2
+ echo "Starting cat $_file"
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool cat $_file
+
elif [ "$1" == "reset" ]
then
@@ -61,6 +82,9 @@ else
echo "usage: metastore.sh backup"
echo " metastore.sh reset"
echo " metastore.sh restore PATH_TO_LOCAL_META"
+ echo " metastore.sh list RESOURCE_PATH"
+ echo " metastore.sh cat RESOURCE_PATH"
+ echo " metastore.sh remove RESOURCE_PATH"
echo " metastore.sh clean [--delete true]"
exit 1
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/17c33dc4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 489e45a..2e1d527 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -41,6 +41,9 @@ public class ResourceTool {
System.out.println("Usage: ResourceTool list RESOURCE_PATH");
System.out.println("Usage: ResourceTool download LOCAL_DIR");
System.out.println("Usage: ResourceTool upload LOCAL_DIR");
+ System.out.println("Usage: ResourceTool reset");
+ System.out.println("Usage: ResourceTool remove RESOURCE_PATH");
+ System.out.println("Usage: ResourceTool cat RESOURCE_PATH");
return;
}
[08/50] [abbrv] kylin git commit: very minor, eclipse settings update
Posted by sh...@apache.org.
very minor, eclipse settings update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11dc04b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11dc04b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11dc04b6
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 11dc04b6117c9a146dc54f919cee0354ceac1e24
Parents: ecd1266
Author: Yang Li <li...@apache.org>
Authored: Sat Mar 5 08:05:37 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Mar 5 08:05:37 2016 +0800
----------------------------------------------------------------------
core-cube/.settings/org.eclipse.core.resources.prefs | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/11dc04b6/core-cube/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/core-cube/.settings/org.eclipse.core.resources.prefs b/core-cube/.settings/org.eclipse.core.resources.prefs
index 04cfa2c..365bbd6 100644
--- a/core-cube/.settings/org.eclipse.core.resources.prefs
+++ b/core-cube/.settings/org.eclipse.core.resources.prefs
@@ -2,5 +2,4 @@ eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
[27/50] [abbrv] kylin git commit: test case for ssb
Posted by sh...@apache.org.
test case for ssb
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8590b2d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8590b2d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8590b2d2
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 8590b2d24f00c5495d0dd6608743ba2bfe431d83
Parents: b7db093
Author: shaofengshi <sh...@apache.org>
Authored: Thu Mar 10 17:43:41 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 10 19:47:02 2016 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/InMemCubeBuilderTest.java | 12 +-
examples/test_case_data/localmeta/cube/ssb.json | 16 ++
.../test_case_data/localmeta/cube_desc/ssb.json | 177 +++++++++++++++++++
...ediate_ssb_19920101000000_19920201000000.csv | 100 +++++++++++
.../localmeta/model_desc/ssb.json | 65 +++++++
.../localmeta/table/SSB.CUSTOMER.json | 41 +++++
.../localmeta/table/SSB.DATES.json | 77 ++++++++
.../localmeta/table/SSB.PART.json | 45 +++++
.../localmeta/table/SSB.SUPPLIER.json | 37 ++++
.../localmeta/table/SSB.V_LINEORDER.json | 85 +++++++++
10 files changed, 650 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index b593e48..76674d3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -61,8 +61,8 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
private static final Logger logger = LoggerFactory.getLogger(InMemCubeBuilderTest.class);
- private static final int INPUT_ROWS = 70000;
- private static final int THREADS = 4;
+ private static final int INPUT_ROWS = 1000;
+ private static final int THREADS = 1;
private static CubeInstance cube;
private static String flatTable;
@@ -75,8 +75,8 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv";
+ cube = cubeManager.getCube("ssb");
+ flatTable = "../examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv";
dictionaryMap = getDictionaryMap(cube, flatTable);
}
@@ -213,7 +213,9 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
List<String> lines = FileUtils.readLines(new File(flatTable), "UTF-8");
for (String line : lines) {
String[] row = line.trim().split(",");
- assert row.length == nColumns;
+ if ( row.length != nColumns) {
+ throw new IllegalStateException();
+ }
if (row[c] != null) {
result.add(Bytes.toBytes(row[c]));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/cube/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ssb.json b/examples/test_case_data/localmeta/cube/ssb.json
new file mode 100644
index 0000000..879754e
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb.json
@@ -0,0 +1,16 @@
+{
+ "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c69",
+ "version" : "2.1",
+ "name" : "ssb",
+ "owner" : "ADMIN",
+ "cost" : 50,
+ "status" : "DISABLED",
+ "segments" : [ {
+ } ],
+ "last_modified" : 1457534216410,
+ "descriptor" : "ssb",
+ "create_time_utc" : 1457444500888,
+ "size_kb" : 0,
+ "input_records_count" : 0,
+ "input_records_size" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
new file mode 100644
index 0000000..7fa0e9a
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -0,0 +1,177 @@
+{
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
+ "version" : "2.1",
+ "name" : "ssb",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "C_CITY",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_CITY",
+ "derived" : null
+ }, {
+ "name" : "C_REGION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_REGION",
+ "derived" : null
+ }, {
+ "name" : "C_NATION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_NATION",
+ "derived" : null
+ }, {
+ "name" : "S_CITY",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_CITY",
+ "derived" : null
+ }, {
+ "name" : "S_REGION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_REGION",
+ "derived" : null
+ }, {
+ "name" : "S_NATION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_V_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "V_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : "5iV8LVYs+PmVUju8QNQ5TQ==",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 694224000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : { }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv b/examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv
new file mode 100644
index 0000000..407074f
--- /dev/null
+++ b/examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv
@@ -0,0 +1,100 @@
+54540,JORDAN 8,MIDDLE EAST,JORDAN,EGYPT 1,MIDDLE EAST,EGYPT,1992,Jan1992,199201,4,3833495,89672,20176290
+304647,JORDAN 8,MIDDLE EAST,JORDAN,IRAQ 5,MIDDLE EAST,IRAQ,1992,Jan1992,199201,3,2081053,99097,23122820
+891817,JORDAN 8,MIDDLE EAST,JORDAN,ALGERIA 1,AFRICA,ALGERIA,1992,Jan1992,199201,4,5670493,108526,29844705
+756262,JORDAN 8,MIDDLE EAST,JORDAN,CHINA 9,ASIA,CHINA,1992,Jan1992,199201,4,6129769,79093,46138050
+666351,JORDAN 8,MIDDLE EAST,JORDAN,RUSSIA 3,EUROPE,RUSSIA,1992,Jan1992,199201,2,4165365,79039,31352216
+835761,JORDAN 8,MIDDLE EAST,JORDAN,MOROCCO 3,AFRICA,MOROCCO,1992,Jan1992,199201,4,1628851,101803,6786880
+359288,JORDAN 8,MIDDLE EAST,JORDAN,EGYPT 4,MIDDLE EAST,EGYPT,1992,Jan1992,199201,2,6014213,80836,45268272
+718610,JORDAN 8,MIDDLE EAST,JORDAN,PERU 1,AMERICA,PERU,1992,Jan1992,199201,2,5543686,97714,48205968
+543324,JORDAN 8,MIDDLE EAST,JORDAN,CANADA 2,AMERICA,CANADA,1992,Jan1992,199201,2,6029793,82038,12305700
+642580,JORDAN 8,MIDDLE EAST,JORDAN,MOROCCO 9,AFRICA,MOROCCO,1992,Jan1992,199201,4,1096236,91353,12180400
+488134,JORDAN 8,MIDDLE EAST,JORDAN,INDONESIA5,ASIA,INDONESIA,1992,Jan1992,199201,4,4852003,67326,42191336
+600300,JORDAN 8,MIDDLE EAST,JORDAN,PERU 4,AMERICA,PERU,1992,Jan1992,199201,3,3948888,72016,25205670
+629068,JORDAN 8,MIDDLE EAST,JORDAN,INDIA 3,ASIA,INDIA,1992,Jan1992,199201,4,493529,59821,498515
+662324,JORDAN 8,MIDDLE EAST,JORDAN,CHINA 4,ASIA,CHINA,1992,Jan1992,199201,2,3668499,77177,31899992
+27105,JORDAN 8,MIDDLE EAST,JORDAN,JORDAN 9,MIDDLE EAST,JORDAN,1992,Jan1992,199201,2,1201364,61926,3715560
+230406,JORDAN 8,MIDDLE EAST,JORDAN,BRAZIL 6,AMERICA,BRAZIL,1992,Jan1992,199201,3,3231391,80183,24322298
+636404,JORDAN 8,MIDDLE EAST,JORDAN,MOZAMBIQU5,AFRICA,MOZAMBIQUE,1992,Jan1992,199201,2,5220741,80422,27477585
+797334,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,JORDAN 5,MIDDLE EAST,JORDAN,1992,Jan1992,199201,4,2665080,85878,5438940
+636211,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,VIETNAM 6,ASIA,VIETNAM,1992,Jan1992,199201,4,4339781,68830,13422006
+412854,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,JAPAN 3,ASIA,JAPAN,1992,Jan1992,199201,4,7065553,106009,53181583
+403310,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,IRAQ 6,MIDDLE EAST,IRAQ,1992,Jan1992,199201,2,798344,72797,5095818
+546702,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,FRANCE 8,EUROPE,FRANCE,1992,Jan1992,199201,4,5945512,104920,0
+831954,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,CHINA 5,ASIA,CHINA,1992,Jan1992,199201,4,8241426,113154,43375930
+15661,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,KENYA 1,AFRICA,KENYA,1992,Jan1992,199201,1,7492288,94599,7567968
+831084,SAUDI ARA4,MIDDLE EAST,SAUDI ARABIA,FRANCE 2,EUROPE,FRANCE,1992,Jan1992,199201,1,2194516,60902,14007552
+283562,ROMANIA 7,EUROPE,ROMANIA,JAPAN 4,ASIA,JAPAN,1992,Jan1992,199201,1,6329027,92733,62594775
+639460,ROMANIA 7,EUROPE,ROMANIA,GERMANY 1,EUROPE,GERMANY,1992,Jan1992,199201,1,1234297,83965,2518974
+263842,ROMANIA 7,EUROPE,ROMANIA,ALGERIA 3,AFRICA,ALGERIA,1992,Jan1992,199201,3,902915,108349,0
+332645,ROMANIA 7,EUROPE,ROMANIA,INDIA 8,ASIA,INDIA,1992,Jan1992,199201,2,6509204,100657,20131560
+957058,ROMANIA 7,EUROPE,ROMANIA,CHINA 0,ASIA,CHINA,1992,Jan1992,199201,3,1555438,66900,11707605
+226995,ROMANIA 7,EUROPE,ROMANIA,KENYA 5,AFRICA,KENYA,1992,Jan1992,199201,2,3974654,115318,25370136
+393867,ROMANIA 7,EUROPE,ROMANIA,ALGERIA 7,AFRICA,ALGERIA,1992,Jan1992,199201,1,4239357,117651,27059730
+998179,ROMANIA 7,EUROPE,ROMANIA,IRAN 7,MIDDLE EAST,IRAN,1992,Jan1992,199201,2,3592566,76627,11111031
+383783,ROMANIA 7,EUROPE,ROMANIA,JAPAN 2,ASIA,JAPAN,1992,Jan1992,199201,1,7370007,112006,47042604
+387948,ROMANIA 7,EUROPE,ROMANIA,BRAZIL 7,AMERICA,BRAZIL,1992,Jan1992,199201,2,9776535,122155,19952114
+379515,ROMANIA 7,EUROPE,ROMANIA,ARGENTINA2,AMERICA,ARGENTINA,1992,Jan1992,199201,3,5190097,95670,39065250
+505473,ROMANIA 7,EUROPE,ROMANIA,ARGENTINA3,AMERICA,ARGENTINA,1992,Jan1992,199201,3,2018084,88707,19959075
+423210,ROMANIA 7,EUROPE,ROMANIA,GERMANY 5,EUROPE,GERMANY,1992,Jan1992,199201,1,322959,67991,1699785
+279170,ROMANIA 7,EUROPE,ROMANIA,IRAQ 5,MIDDLE EAST,IRAQ,1992,Jan1992,199201,2,4914957,68949,48609468
+856452,ROMANIA 7,EUROPE,ROMANIA,RUSSIA 3,EUROPE,RUSSIA,1992,Jan1992,199201,1,3309763,84504,21126150
+816854,ROMANIA 7,EUROPE,ROMANIA,ROMANIA 8,EUROPE,ROMANIA,1992,Jan1992,199201,1,2443717,106248,21249720
+678247,ROMANIA 7,EUROPE,ROMANIA,UNITED KI0,EUROPE,UNITED KINGDOM,1992,Jan1992,199201,1,367563,73512,0
+98372,ROMANIA 7,EUROPE,ROMANIA,FRANCE 0,EUROPE,FRANCE,1992,Jan1992,199201,2,4834665,82222,9866664
+240580,ROMANIA 7,EUROPE,ROMANIA,SAUDI ARA6,MIDDLE EAST,SAUDI ARABIA,1992,Jan1992,199201,1,2744628,91234,14445415
+547655,CHINA 4,ASIA,CHINA,PERU 9,AMERICA,PERU,1992,Jan1992,199201,1,7127209,102157,70488882
+380567,CHINA 4,ASIA,CHINA,PERU 2,AMERICA,PERU,1992,Jan1992,199201,1,3196247,98853,9885300
+266841,CHINA 4,ASIA,CHINA,UNITED ST7,AMERICA,UNITED STATES,1992,Jan1992,199201,1,166320,108469,1446264
+720209,CHINA 4,ASIA,CHINA,UNITED KI9,EUROPE,UNITED KINGDOM,1992,Jan1992,199201,1,5310014,73750,59000160
+751631,CHINA 4,ASIA,CHINA,GERMANY 0,EUROPE,GERMANY,1992,Jan1992,199201,3,2190745,100956,16489480
+843820,CHINA 4,ASIA,CHINA,MOZAMBIQU5,AFRICA,MOZAMBIQUE,1992,Jan1992,199201,1,5587655,105826,23281896
+30096,CHINA 4,ASIA,CHINA,EGYPT 4,MIDDLE EAST,EGYPT,1992,Jan1992,199201,3,2216354,61565,24626160
+675496,CHINA 4,ASIA,CHINA,KENYA 6,AFRICA,KENYA,1992,Jan1992,199201,3,5731336,88287,30164930
+433088,CHINA 4,ASIA,CHINA,INDIA 6,ASIA,INDIA,1992,Jan1992,199201,1,3920870,61263,16336960
+897776,CHINA 4,ASIA,CHINA,JAPAN 9,ASIA,JAPAN,1992,Jan1992,199201,1,8430538,106423,26073831
+784370,CHINA 4,ASIA,CHINA,MOROCCO 0,AFRICA,MOROCCO,1992,Jan1992,199201,1,820247,87260,5235624
+23545,CHINA 4,ASIA,CHINA,BRAZIL 4,AMERICA,BRAZIL,1992,Jan1992,199201,1,4593593,88112,39944288
+296221,CHINA 4,ASIA,CHINA,FRANCE 1,EUROPE,FRANCE,1992,Jan1992,199201,4,1601848,73032,10224564
+706630,CHINA 4,ASIA,CHINA,IRAN 6,MIDDLE EAST,IRAN,1992,Jan1992,199201,3,294588,98196,3273200
+697821,CHINA 4,ASIA,CHINA,EGYPT 7,MIDDLE EAST,EGYPT,1992,Jan1992,199201,1,3721244,109127,28009366
+596762,CHINA 4,ASIA,CHINA,GERMANY 1,EUROPE,GERMANY,1992,Jan1992,199201,2,184015,111524,185874
+659386,CHINA 4,ASIA,CHINA,GERMANY 7,EUROPE,GERMANY,1992,Jan1992,199201,4,3462930,80721,3497910
+184270,CHINA 4,ASIA,CHINA,CANADA 3,AMERICA,CANADA,1992,Jan1992,199201,1,4313349,81256,42659505
+720238,CHINA 4,ASIA,CHINA,ETHIOPIA 2,AFRICA,ETHIOPIA,1992,Jan1992,199201,3,249123,75492,251640
+27452,CHINA 4,ASIA,CHINA,ALGERIA 0,AFRICA,ALGERIA,1992,Jan1992,199201,1,4950846,82767,15311895
+305657,CHINA 4,ASIA,CHINA,INDIA 4,ASIA,INDIA,1992,Jan1992,199201,3,6733692,99758,74818800
+648403,CHINA 4,ASIA,CHINA,JAPAN 3,ASIA,JAPAN,1992,Jan1992,199201,3,5112232,81082,15811029
+176995,CHINA 4,ASIA,CHINA,INDONESIA7,ASIA,INDONESIA,1992,Jan1992,199201,3,5202766,124319,39160611
+70310,CHINA 4,ASIA,CHINA,UNITED KI6,EUROPE,UNITED KINGDOM,1992,Jan1992,199201,1,4711540,76818,40969920
+255617,CHINA 4,ASIA,CHINA,MOROCCO 9,AFRICA,MOROCCO,1992,Jan1992,199201,2,4560540,94356,0
+75211,CHINA 4,ASIA,CHINA,IRAN 7,MIDDLE EAST,IRAN,1992,Jan1992,199201,1,3345112,71172,21351780
+816962,CHINA 4,ASIA,CHINA,FRANCE 0,EUROPE,FRANCE,1992,Jan1992,199201,3,1037163,112735,9018816
+316538,CHINA 4,ASIA,CHINA,SAUDI ARA1,MIDDLE EAST,SAUDI ARABIA,1992,Jan1992,199201,3,572063,93271,4974464
+149141,CHINA 4,ASIA,CHINA,VIETNAM 5,ASIA,VIETNAM,1992,Jan1992,199201,2,1006858,71408,6426756
+115842,CANADA 4,AMERICA,CANADA,RUSSIA 2,EUROPE,RUSSIA,1992,Jan1992,199201,5,6599047,111470,27496032
+777719,CANADA 4,AMERICA,CANADA,ARGENTINA7,AMERICA,ARGENTINA,1992,Jan1992,199201,5,2730953,107800,14373440
+119251,CANADA 4,AMERICA,CANADA,FRANCE 2,EUROPE,FRANCE,1992,Jan1992,199201,3,4066070,76215,12575475
+944695,CANADA 4,AMERICA,CANADA,VIETNAM 8,ASIA,VIETNAM,1992,Jan1992,199201,5,6544563,104379,6610670
+718372,CANADA 4,AMERICA,CANADA,UNITED KI3,EUROPE,UNITED KINGDOM,1992,Jan1992,199201,5,5943703,83420,31282650
+523579,CANADA 4,AMERICA,CANADA,UNITED ST5,AMERICA,UNITED STATES,1992,Jan1992,199201,2,6929426,96153,44230380
+668042,CANADA 4,AMERICA,CANADA,IRAQ 4,MIDDLE EAST,IRAQ,1992,Jan1992,199201,5,1484714,60600,3030030
+235495,CANADA 4,AMERICA,CANADA,INDONESIA9,ASIA,INDONESIA,1992,Jan1992,199201,3,823956,85828,3433152
+59728,CANADA 4,AMERICA,CANADA,PERU 0,AMERICA,PERU,1992,Jan1992,199201,3,7608241,101263,66158624
+951869,CANADA 4,AMERICA,CANADA,ALGERIA 4,AFRICA,ALGERIA,1992,Jan1992,199201,3,2297300,115249,19976528
+527596,RUSSIA 8,EUROPE,RUSSIA,SAUDI ARA7,MIDDLE EAST,SAUDI ARABIA,1992,Jan1992,199201,3,1045579,97414,9091992
+11320,RUSSIA 8,EUROPE,RUSSIA,MOZAMBIQU0,AFRICA,MOZAMBIQUE,1992,Jan1992,199201,3,4658083,73879,14406444
+355756,RUSSIA 8,EUROPE,RUSSIA,ALGERIA 9,AFRICA,ALGERIA,1992,Jan1992,199201,4,6853812,108704,21197358
+292956,RUSSIA 8,EUROPE,RUSSIA,BRAZIL 7,AMERICA,BRAZIL,1992,Jan1992,199201,3,4437736,116936,4482562
+429513,RUSSIA 8,EUROPE,RUSSIA,UNITED ST3,AMERICA,UNITED STATES,1992,Jan1992,199201,5,3917802,86549,12116916
+728473,RUSSIA 8,EUROPE,RUSSIA,MOROCCO 1,AFRICA,MOROCCO,1992,Jan1992,199201,5,4092925,90086,26125056
+83255,RUSSIA 8,EUROPE,RUSSIA,CANADA 6,AMERICA,CANADA,1992,Jan1992,199201,2,3095625,74295,0
+265369,RUSSIA 8,EUROPE,RUSSIA,ARGENTINA3,AMERICA,ARGENTINA,1992,Jan1992,199201,2,6342165,80061,19614945
+478118,RUSSIA 8,EUROPE,RUSSIA,PERU 7,AMERICA,PERU,1992,Jan1992,199201,5,4557542,65765,4603578
+85918,RUSSIA 8,EUROPE,RUSSIA,EGYPT 1,MIDDLE EAST,EGYPT,1992,Jan1992,199201,3,519767,114234,5140557
+694781,RUSSIA 8,EUROPE,RUSSIA,PERU 9,AMERICA,PERU,1992,Jan1992,199201,3,843481,106545,4439375
+810359,RUSSIA 8,EUROPE,RUSSIA,CANADA 6,AMERICA,CANADA,1992,Jan1992,199201,5,5666199,76158,42648816
+248440,RUSSIA 8,EUROPE,RUSSIA,UNITED KI6,EUROPE,UNITED KINGDOM,1992,Jan1992,199201,3,1616132,83305,4998348
+969066,RUSSIA 8,EUROPE,RUSSIA,RUSSIA 0,EUROPE,RUSSIA,1992,Jan1992,199201,2,3523102,68101,10896192
+377269,MOZAMBIQU4,AFRICA,MOZAMBIQUE,MOROCCO 2,AFRICA,MOROCCO,1992,Jan1992,199201,2,1466066,80775,1480875
+744978,MOZAMBIQU4,AFRICA,MOZAMBIQUE,MOROCCO 9,AFRICA,MOROCCO,1992,Jan1992,199201,2,196225,121376,606882
+931091,MOZAMBIQU4,AFRICA,MOZAMBIQUE,RUSSIA 6,EUROPE,RUSSIA,1992,Jan1992,199201,4,769726,67323,1570870
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/model_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ssb.json b/examples/test_case_data/localmeta/model_desc/ssb.json
new file mode 100644
index 0000000..e8c0de0
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/ssb.json
@@ -0,0 +1,65 @@
+{
+ "uuid" : "cd92588f-b987-4a12-b90f-e32c44345c64",
+ "version" : "2.1",
+ "name" : "ssb",
+ "description" : "",
+ "lookups" : [ {
+ "table" : "SSB.PART",
+ "join" : {
+ "type" : "left",
+ "primary_key" : [ "P_PARTKEY" ],
+ "foreign_key" : [ "LO_PARTKEY" ]
+ }
+ }, {
+ "table" : "SSB.CUSTOMER",
+ "join" : {
+ "type" : "left",
+ "primary_key" : [ "C_CUSTKEY" ],
+ "foreign_key" : [ "LO_CUSTKEY" ]
+ }
+ }, {
+ "table" : "SSB.SUPPLIER",
+ "join" : {
+ "type" : "left",
+ "primary_key" : [ "S_SUPPKEY" ],
+ "foreign_key" : [ "LO_SUPPKEY" ]
+ }
+ }, {
+ "table" : "SSB.DATES",
+ "join" : {
+ "type" : "left",
+ "primary_key" : [ "D_DATEKEY" ],
+ "foreign_key" : [ "LO_ORDERDATE" ]
+ }
+ } ],
+ "dimensions" : [ {
+ "table" : "SSB.V_LINEORDER",
+ "columns" : [ "LO_DATE" ]
+ }, {
+ "table" : "SSB.PART",
+ "columns" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "table" : "SSB.CUSTOMER",
+ "columns" : [ "C_CITY", "C_NATION", "C_REGION" ]
+ }, {
+ "table" : "SSB.SUPPLIER",
+ "columns" : [ "S_CITY", "S_NATION", "S_REGION" ]
+ }, {
+ "table" : "SSB.DATES",
+ "columns" : [ "D_YEAR", "D_YEARMONTHNUM", "D_YEARMONTH", "D_WEEKNUMINYEAR" ]
+ } ],
+ "metrics" : [ "LO_REVENUE", "LO_SUPPLYCOST", "V_REVENUE" ],
+ "capacity" : "MEDIUM",
+ "last_modified" : 1457444314662,
+ "fact_table" : "SSB.V_LINEORDER",
+ "filter_condition" : "",
+ "partition_desc" : {
+ "partition_date_column" : "SSB.V_LINEORDER.LO_DATE",
+ "partition_time_column" : null,
+ "partition_date_start" : 0,
+ "partition_date_format" : "yyyyMMdd",
+ "partition_time_format" : "HH:mm:ss",
+ "partition_type" : "APPEND",
+ "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/table/SSB.CUSTOMER.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SSB.CUSTOMER.json b/examples/test_case_data/localmeta/table/SSB.CUSTOMER.json
new file mode 100644
index 0000000..ab92352
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SSB.CUSTOMER.json
@@ -0,0 +1,41 @@
+{
+ "uuid" : "d70320ec-949f-44df-8bf4-92dc005dd07d",
+ "version" : "2.1",
+ "name" : "CUSTOMER",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "C_CUSTKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "2",
+ "name" : "C_NAME",
+ "datatype" : "varchar(25)"
+ }, {
+ "id" : "3",
+ "name" : "C_ADDRESS",
+ "datatype" : "varchar(40)"
+ }, {
+ "id" : "4",
+ "name" : "C_CITY",
+ "datatype" : "varchar(10)"
+ }, {
+ "id" : "5",
+ "name" : "C_NATION",
+ "datatype" : "varchar(15)"
+ }, {
+ "id" : "6",
+ "name" : "C_REGION",
+ "datatype" : "varchar(12)"
+ }, {
+ "id" : "7",
+ "name" : "C_PHONE",
+ "datatype" : "varchar(15)"
+ }, {
+ "id" : "8",
+ "name" : "C_MKTSEGMENT",
+ "datatype" : "varchar(10)"
+ } ],
+ "database" : "SSB",
+ "last_modified" : 1457444146362,
+ "source_type" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/table/SSB.DATES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SSB.DATES.json b/examples/test_case_data/localmeta/table/SSB.DATES.json
new file mode 100644
index 0000000..ed93c2d
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SSB.DATES.json
@@ -0,0 +1,77 @@
+{
+ "uuid" : "38d2a2ee-2968-4f09-94b5-6925ba27a699",
+ "version" : "2.1",
+ "name" : "DATES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "D_DATEKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "2",
+ "name" : "D_DATE",
+ "datatype" : "varchar(18)"
+ }, {
+ "id" : "3",
+ "name" : "D_DAYOFWEEK",
+ "datatype" : "varchar(8)"
+ }, {
+ "id" : "4",
+ "name" : "D_MONTH",
+ "datatype" : "varchar(9)"
+ }, {
+ "id" : "5",
+ "name" : "D_YEAR",
+ "datatype" : "integer"
+ }, {
+ "id" : "6",
+ "name" : "D_YEARMONTHNUM",
+ "datatype" : "integer"
+ }, {
+ "id" : "7",
+ "name" : "D_YEARMONTH",
+ "datatype" : "varchar(7)"
+ }, {
+ "id" : "8",
+ "name" : "D_DAYNUMINWEEK",
+ "datatype" : "integer"
+ }, {
+ "id" : "9",
+ "name" : "D_DAYNUMINMONTH",
+ "datatype" : "integer"
+ }, {
+ "id" : "10",
+ "name" : "D_DAYNUMINYEAR",
+ "datatype" : "integer"
+ }, {
+ "id" : "11",
+ "name" : "D_MONTHNUMINYEAR",
+ "datatype" : "integer"
+ }, {
+ "id" : "12",
+ "name" : "D_WEEKNUMINYEAR",
+ "datatype" : "integer"
+ }, {
+ "id" : "13",
+ "name" : "D_SELLINGSEASON",
+ "datatype" : "varchar(12)"
+ }, {
+ "id" : "14",
+ "name" : "D_LASTDAYINWEEKFL",
+ "datatype" : "integer"
+ }, {
+ "id" : "15",
+ "name" : "D_LASTDAYINMONTHFL",
+ "datatype" : "integer"
+ }, {
+ "id" : "16",
+ "name" : "D_HOLIDAYFL",
+ "datatype" : "integer"
+ }, {
+ "id" : "17",
+ "name" : "D_WEEKDAYFL",
+ "datatype" : "integer"
+ } ],
+ "database" : "SSB",
+ "last_modified" : 1457444145262,
+ "source_type" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/table/SSB.PART.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SSB.PART.json b/examples/test_case_data/localmeta/table/SSB.PART.json
new file mode 100644
index 0000000..7b3230e
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SSB.PART.json
@@ -0,0 +1,45 @@
+{
+ "uuid" : "e82e7622-ff20-4f0a-9d0d-75524034b304",
+ "version" : "2.1",
+ "name" : "PART",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "P_PARTKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "2",
+ "name" : "P_NAME",
+ "datatype" : "varchar(22)"
+ }, {
+ "id" : "3",
+ "name" : "P_MFGR",
+ "datatype" : "varchar(6)"
+ }, {
+ "id" : "4",
+ "name" : "P_CATEGORY",
+ "datatype" : "varchar(7)"
+ }, {
+ "id" : "5",
+ "name" : "P_BRAND",
+ "datatype" : "varchar(9)"
+ }, {
+ "id" : "6",
+ "name" : "P_COLOR",
+ "datatype" : "varchar(11)"
+ }, {
+ "id" : "7",
+ "name" : "P_TYPE",
+ "datatype" : "varchar(25)"
+ }, {
+ "id" : "8",
+ "name" : "P_SIZE",
+ "datatype" : "integer"
+ }, {
+ "id" : "9",
+ "name" : "P_CONTAINER",
+ "datatype" : "varchar(10)"
+ } ],
+ "database" : "SSB",
+ "last_modified" : 1457444145980,
+ "source_type" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/table/SSB.SUPPLIER.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SSB.SUPPLIER.json b/examples/test_case_data/localmeta/table/SSB.SUPPLIER.json
new file mode 100644
index 0000000..ba6931e
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SSB.SUPPLIER.json
@@ -0,0 +1,37 @@
+{
+ "uuid" : "96b155b7-89e9-42dc-8621-d4bbae823b70",
+ "version" : "2.1",
+ "name" : "SUPPLIER",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "S_SUPPKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "2",
+ "name" : "S_NAME",
+ "datatype" : "varchar(25)"
+ }, {
+ "id" : "3",
+ "name" : "S_ADDRESS",
+ "datatype" : "varchar(25)"
+ }, {
+ "id" : "4",
+ "name" : "S_CITY",
+ "datatype" : "varchar(10)"
+ }, {
+ "id" : "5",
+ "name" : "S_NATION",
+ "datatype" : "varchar(15)"
+ }, {
+ "id" : "6",
+ "name" : "S_REGION",
+ "datatype" : "varchar(12)"
+ }, {
+ "id" : "7",
+ "name" : "S_PHONE",
+ "datatype" : "varchar(15)"
+ } ],
+ "database" : "SSB",
+ "last_modified" : 1457444146835,
+ "source_type" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8590b2d2/examples/test_case_data/localmeta/table/SSB.V_LINEORDER.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SSB.V_LINEORDER.json b/examples/test_case_data/localmeta/table/SSB.V_LINEORDER.json
new file mode 100644
index 0000000..4c9c8d6
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SSB.V_LINEORDER.json
@@ -0,0 +1,85 @@
+{
+ "uuid" : "b017d54b-e7b7-465b-a4db-b47f68baf1ad",
+ "version" : "2.1",
+ "name" : "V_LINEORDER",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LO_ORDERKEY",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LO_LINENUMBER",
+ "datatype" : "bigint"
+ }, {
+ "id" : "3",
+ "name" : "LO_CUSTKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "4",
+ "name" : "LO_PARTKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "5",
+ "name" : "LO_SUPPKEY",
+ "datatype" : "integer"
+ }, {
+ "id" : "6",
+ "name" : "LO_DATE",
+ "datatype" : "varchar(256)"
+ }, {
+ "id" : "7",
+ "name" : "LO_ORDERDATE",
+ "datatype" : "integer"
+ }, {
+ "id" : "8",
+ "name" : "LO_ORDERPRIOTITY",
+ "datatype" : "varchar(15)"
+ }, {
+ "id" : "9",
+ "name" : "LO_SHIPPRIOTITY",
+ "datatype" : "integer"
+ }, {
+ "id" : "10",
+ "name" : "LO_QUANTITY",
+ "datatype" : "bigint"
+ }, {
+ "id" : "11",
+ "name" : "LO_EXTENDEDPRICE",
+ "datatype" : "bigint"
+ }, {
+ "id" : "12",
+ "name" : "LO_ORDTOTALPRICE",
+ "datatype" : "bigint"
+ }, {
+ "id" : "13",
+ "name" : "LO_DISCOUNT",
+ "datatype" : "bigint"
+ }, {
+ "id" : "14",
+ "name" : "LO_REVENUE",
+ "datatype" : "bigint"
+ }, {
+ "id" : "15",
+ "name" : "LO_SUPPLYCOST",
+ "datatype" : "bigint"
+ }, {
+ "id" : "16",
+ "name" : "LO_TAX",
+ "datatype" : "bigint"
+ }, {
+ "id" : "17",
+ "name" : "LO_COMMITDATE",
+ "datatype" : "integer"
+ }, {
+ "id" : "18",
+ "name" : "LO_SHIPMODE",
+ "datatype" : "varchar(10)"
+ }, {
+ "id" : "19",
+ "name" : "V_REVENUE",
+ "datatype" : "bigint"
+ } ],
+ "database" : "SSB",
+ "last_modified" : 1457444145578,
+ "source_type" : 0
+}
\ No newline at end of file
[12/50] [abbrv] kylin git commit: KYLIN-1420 enhance and update test
case
Posted by sh...@apache.org.
KYLIN-1420 enhance and update test case
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/250978d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/250978d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/250978d8
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 250978d887e577582f26fc05036d6a4af82dfd0b
Parents: 3736f72
Author: shaofengshi <sh...@apache.org>
Authored: Mon Feb 15 18:06:18 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:15:11 2016 +0800
----------------------------------------------------------------------
.../kylin/gridtable/GTScanRangePlanner.java | 27 ++++++++------------
.../kylin/gridtable/DictGridTableTest.java | 2 +-
2 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/250978d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index d314dde..559a245 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -1,17 +1,8 @@
package org.apache.kylin.gridtable;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -26,9 +17,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import java.util.*;
public class GTScanRangePlanner {
@@ -108,8 +97,10 @@ public class GTScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (partitionColRef != null && range.column.equals(partitionColRef)) {
if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 //
- && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) <= 0) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <=.
+ && (rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0 //
+ || rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) == 0 //
+ && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}",//
new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()), makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
@@ -346,9 +337,11 @@ public class GTScanRangePlanner {
private ByteArray begin = ByteArray.EMPTY;
private ByteArray end = ByteArray.EMPTY;
private Set<ByteArray> valueSet;
+ private FilterOperatorEnum op;
public ColumnRange(TblColRef column, Set<ByteArray> values, FilterOperatorEnum op) {
this.column = column;
+ this.op = op;
switch (op) {
case EQ:
http://git-wip-us.apache.org/repos/asf/kylin/blob/250978d8/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index df69c17..674aa15 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -118,7 +118,7 @@ public class DictGridTableTest {
{
LogicalTupleFilter filter = and(timeComp4, ageComp1);
List<GTScanRange> r = planner.planScanRanges(filter);
- assertEquals(0, r.size());
+ assertEquals(1, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
[13/50] [abbrv] kylin git commit: KYLIN-1387 Streaming cubing doesn't
generate cuboids files on HDFS, cause cube merge failure
Posted by sh...@apache.org.
KYLIN-1387 Streaming cubing doesn't generate cuboids files on HDFS, cause cube merge failure
Conflicts:
engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/929c7a49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/929c7a49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/929c7a49
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 929c7a4908a3cd655ab31e71eb6453971f3acd36
Parents: 250978d
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 2 17:34:46 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:19:36 2016 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/CompoundCuboidWriter.java | 57 ++++++++++++++
.../kylin/cube/inmemcubing/ICuboidWriter.java | 4 +-
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 81 ++++++++++++++++++++
.../mr/steps/MapContextGTRecordWriter.java | 68 ++--------------
.../streaming/cube/StreamingCubeBuilder.java | 12 ++-
.../storage/hbase/steps/HBaseCuboidWriter.java | 24 +++---
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 2 +-
.../hbase/steps/HBaseStreamingOutput.java | 8 +-
.../hbase/steps/SequenceFileCuboidWriter.java | 75 ++++++++++++++++++
11 files changed, 254 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
new file mode 100644
index 0000000..46eef50
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/CompoundCuboidWriter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.inmemcubing;
+
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.io.IOException;
+
+/**
+ */
+public class CompoundCuboidWriter implements ICuboidWriter {
+
+ private Iterable<ICuboidWriter> cuboidWriters;
+
+ public CompoundCuboidWriter(Iterable<ICuboidWriter> cuboidWriters) {
+ this.cuboidWriters = cuboidWriters;
+
+ }
+
+ @Override
+ public void write(long cuboidId, GTRecord record) throws IOException {
+ for (ICuboidWriter writer : cuboidWriters) {
+ writer.write(cuboidId, record);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ for (ICuboidWriter writer : cuboidWriters) {
+ writer.flush();
+ }
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ for (ICuboidWriter writer : cuboidWriters) {
+ writer.close();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
index 9e26e5e..e6cfa02 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java
@@ -27,7 +27,7 @@ public interface ICuboidWriter {
void write(long cuboidId, GTRecord record) throws IOException;
- void flush();
+ void flush() throws IOException;
- void close();
+ void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index ba50880..d370b0d 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -56,6 +56,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
+ public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
new file mode 100644
index 0000000..e201705
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/KVGTRecordWriter.java
@@ -0,0 +1,81 @@
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.ByteArrayWritable;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ */
+public abstract class KVGTRecordWriter implements ICuboidWriter {
+
+ private static final Log logger = LogFactory.getLog(KVGTRecordWriter.class);
+ private Long lastCuboidId;
+ protected CubeSegment cubeSegment;
+ protected CubeDesc cubeDesc;
+
+ private AbstractRowKeyEncoder rowKeyEncoder;
+ private int dimensions;
+ private int measureCount;
+ private byte[] keyBuf;
+ private int[] measureColumnsIndex;
+ private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ private ByteArrayWritable outputKey = new ByteArrayWritable();
+ private ByteArrayWritable outputValue = new ByteArrayWritable();
+ private long cuboidRowCount = 0;
+
+ //for shard
+
+ public KVGTRecordWriter(CubeDesc cubeDesc, CubeSegment cubeSegment) {
+ this.cubeDesc = cubeDesc;
+ this.cubeSegment = cubeSegment;
+ this.measureCount = cubeDesc.getMeasures().size();
+ }
+
+ @Override
+ public void write(long cuboidId, GTRecord record) throws IOException {
+
+ if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) {
+ if (lastCuboidId != null) {
+ logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows");
+ cuboidRowCount = 0;
+ }
+ // output another cuboid
+ initVariables(cuboidId);
+ lastCuboidId = cuboidId;
+ }
+
+ cuboidRowCount++;
+ rowKeyEncoder.encode(record, record.getInfo().getPrimaryKey(), keyBuf);
+
+ //output measures
+ valueBuf.clear();
+ record.exportColumns(measureColumnsIndex, valueBuf);
+
+ outputKey.set(keyBuf, 0, keyBuf.length);
+ outputValue.set(valueBuf.array(), 0, valueBuf.position());
+ writeAsKeyValue(outputKey, outputValue);
+ }
+
+ protected abstract void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException;
+
+ private void initVariables(Long cuboidId) {
+ rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
+ keyBuf = rowKeyEncoder.createBuf();
+
+ dimensions = Long.bitCount(cuboidId);
+ measureColumnsIndex = new int[measureCount];
+ for (int i = 0; i < measureCount; i++) {
+ measureColumnsIndex[i] = dimensions + i;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index 8416d95..6b4d07d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -1,76 +1,32 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.BitSet;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
-import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.ByteArrayWritable;
-import org.apache.kylin.gridtable.GTRecord;
+
+import java.io.IOException;
/**
*/
-public class MapContextGTRecordWriter implements ICuboidWriter {
+public class MapContextGTRecordWriter extends KVGTRecordWriter {
private static final Log logger = LogFactory.getLog(MapContextGTRecordWriter.class);
protected MapContext<?, ?, ByteArrayWritable, ByteArrayWritable> mapContext;
- private Long lastCuboidId;
- protected CubeSegment cubeSegment;
- protected CubeDesc cubeDesc;
-
- private AbstractRowKeyEncoder rowKeyEncoder;
- private int dimensions;
- private int measureCount;
- private byte[] keyBuf;
- private int[] measureColumnsIndex;
- private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private ByteArrayWritable outputKey = new ByteArrayWritable();
- private ByteArrayWritable outputValue = new ByteArrayWritable();
- private long cuboidRowCount = 0;
-
- //for shard
public MapContextGTRecordWriter(MapContext<?, ?, ByteArrayWritable, ByteArrayWritable> mapContext, CubeDesc cubeDesc, CubeSegment cubeSegment) {
+ super(cubeDesc, cubeSegment);
this.mapContext = mapContext;
- this.cubeDesc = cubeDesc;
- this.cubeSegment = cubeSegment;
- this.measureCount = cubeDesc.getMeasures().size();
}
@Override
- public void write(long cuboidId, GTRecord record) throws IOException {
-
- if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) {
- if (lastCuboidId != null) {
- logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows");
- cuboidRowCount = 0;
- }
- // output another cuboid
- initVariables(cuboidId);
- lastCuboidId = cuboidId;
- }
-
- cuboidRowCount++;
- rowKeyEncoder.encode(record, record.getInfo().getPrimaryKey(), keyBuf);
-
- //output measures
- valueBuf.clear();
- record.exportColumns(measureColumnsIndex, valueBuf);
-
- outputKey.set(keyBuf, 0, keyBuf.length);
- outputValue.set(valueBuf.array(), 0, valueBuf.position());
+ protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
try {
- mapContext.write(outputKey, outputValue);
+ mapContext.write(key, value);
} catch (InterruptedException e) {
- throw new RuntimeException(e);
+ throw new IOException(e);
}
}
@@ -84,14 +40,4 @@ public class MapContextGTRecordWriter implements ICuboidWriter {
}
- private void initVariables(Long cuboidId) {
- rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, Cuboid.findById(cubeDesc, cuboidId));
- keyBuf = rowKeyEncoder.createBuf();
-
- dimensions = Long.bitCount(cuboidId);
- measureColumnsIndex = new int[measureCount];
- for (int i = 0; i < measureCount; i++) {
- measureColumnsIndex[i] = dimensions + i;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index c4f2b7e..ec2ad91 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -99,6 +99,14 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
+ } catch (IOException e) {
+ throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
+ } finally {
+ try {
+ cuboidWriter.close();
+ } catch (IOException e) {
+ throw new RuntimeException("error build cube from StreamingBatch", e.getCause());
+ }
}
}
@@ -107,7 +115,9 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
try {
- return cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
+ CubeSegment segment = cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
+ segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
+ return segment;
} catch (IOException e) {
throw new RuntimeException("failed to create IBuildable", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index c4dc0b5..ddc868d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -33,9 +33,8 @@
*/
package org.apache.kylin.storage.hbase.steps;
-import java.io.IOException;
-import java.util.List;
-
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
@@ -51,13 +50,14 @@ import org.apache.kylin.gridtable.GTRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
/**
*/
-public final class HBaseCuboidWriter implements ICuboidWriter {
+public class HBaseCuboidWriter implements ICuboidWriter {
- private static final Logger logger = LoggerFactory.getLogger(HBaseStreamingOutput.class);
+ private static final Logger logger = LoggerFactory.getLogger(HBaseCuboidWriter.class);
private static final int BATCH_PUT_THRESHOLD = 10000;
@@ -125,8 +125,8 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
}
}
- public final void flush() {
- try {
+ @Override
+ public final void flush() throws IOException {
if (!puts.isEmpty()) {
long t = System.currentTimeMillis();
if (hTable != null) {
@@ -136,14 +136,12 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
}
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
}
@Override
- public void close() {
-
+ public void close() throws IOException {
+ flush();
+ IOUtils.closeQuietly(hTable);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 4c2737d..7bb3647 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -80,7 +80,7 @@ public class HBaseMROutput2Transition implements IMROutput2 {
@Override
public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow) {
- jobFlow.addTask(steps.createMergeGCStep());
+ steps.addMergingGarbageCollectionSteps(jobFlow);
}
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index 2a21640..a828728 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -161,7 +161,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
toDeletePaths.addAll(getMergingHDFSPaths());
HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep();
- step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+ step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HDFS);
step.setDeletePaths(toDeletePaths);
step.setJobId(jobId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 770be3c..4cc4794 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -18,9 +18,11 @@
package org.apache.kylin.storage.hbase.steps;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,6 +33,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.inmemcubing.CompoundCuboidWriter;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -54,7 +57,10 @@ public class HBaseStreamingOutput implements IStreamingOutput {
final HTableInterface hTable;
hTable = createHTable(cubeSegment);
- return new HBaseCuboidWriter(cubeSegment, hTable);
+ List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
+ cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
+ cuboidWriters.add(new SequenceFileCuboidWriter(cubeSegment.getCubeDesc(), cubeSegment));
+ return new CompoundCuboidWriter(cuboidWriters);
} catch (IOException e) {
throw new RuntimeException("failed to get ICuboidWriter", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/929c7a49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
new file mode 100644
index 0000000..4d76522
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -0,0 +1,75 @@
+package org.apache.kylin.storage.hbase.steps;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.ByteArrayWritable;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.JobBuilderSupport;
+import org.apache.kylin.engine.mr.steps.KVGTRecordWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ */
+public class SequenceFileCuboidWriter extends KVGTRecordWriter {
+
+ private static final Logger logger = LoggerFactory.getLogger(SequenceFileCuboidWriter.class);
+ private SequenceFile.Writer writer = null;
+
+ public SequenceFileCuboidWriter(CubeDesc cubeDesc, CubeSegment segment) {
+ super(cubeDesc, segment);
+ }
+
+
+ @Override
+ protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
+ if (writer == null) {
+ synchronized (SequenceFileCuboidWriter.class) {
+ if (writer == null) {
+ JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
+ String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
+ Path cuboidPath = new Path(cuboidRoot);
+ FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
+ try {
+ if (fs.exists(cuboidPath)) {
+ fs.delete(cuboidPath, true);
+ }
+
+ fs.mkdirs(cuboidPath);
+ } finally {
+ IOUtils.closeQuietly(fs);
+ }
+
+ Path cuboidFile = new Path(cuboidPath, "data.seq");
+ logger.debug("Cuboid is written to " + cuboidFile);
+ writer = SequenceFile.createWriter(HadoopUtil.getCurrentConfiguration(), SequenceFile.Writer.file(cuboidFile), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class));
+ }
+ }
+ }
+
+ Text outputValue = new Text();
+ Text outputKey = new Text();
+ outputKey.set(key.array(), key.offset(), key.length());
+ outputValue.set(value.array(), value.offset(), value.length());
+ writer.append(outputKey, outputValue);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (writer != null) {
+ writer.hflush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeQuietly(writer);
+ }
+}
[15/50] [abbrv] kylin git commit: KYLIN-1421 Cube "source record" is
always zero for streaming
Posted by sh...@apache.org.
KYLIN-1421 Cube "source record" is always zero for streaming
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50aab0b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50aab0b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50aab0b0
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 50aab0b08ea1532998db1f11084c1ae908a42e56
Parents: 9352e5a
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 10:30:05 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:20:05 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/50aab0b0/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index ec2ad91..b4182fe 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -117,6 +117,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
try {
CubeSegment segment = cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
+ segment.setInputRecords(streamingBatch.getMessages().size());
return segment;
} catch (IOException e) {
throw new RuntimeException("failed to create IBuildable", e);
[29/50] [abbrv] kylin git commit: update pom verison number according
to branch renaming
Posted by sh...@apache.org.
update pom verison number according to branch renaming
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3569b22b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3569b22b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3569b22b
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 3569b22bd6ba5f6fc9cbc23eb0085509f98a4de9
Parents: ea4a42c
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 10 20:00:45 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 10 20:00:45 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 2 +-
.../src/main/java/org/apache/kylin/common/KylinVersion.java | 8 ++------
.../kylin/common/persistence/RootPersistentEntity.java | 2 --
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
engine-streaming/pom.xml | 2 +-
examples/sample_cube/metadata/cube/kylin_sales_cube.json | 2 +-
.../metadata/cube_desc/kylin_sales_cube_desc.json | 2 +-
.../sample_cube/metadata/model_desc/kylin_sales_model.json | 2 +-
examples/sample_cube/metadata/project/learn_kylin.json | 2 +-
.../sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json | 2 +-
.../metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 2 +-
examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json | 2 +-
.../cube/test_kylin_cube_with_slr_1_new_segment.json | 2 +-
.../localmeta/cube/test_kylin_cube_with_slr_empty.json | 2 +-
.../cube/test_kylin_cube_with_slr_left_join_empty.json | 2 +-
.../cube/test_kylin_cube_with_slr_left_join_ready.json | 2 +-
.../localmeta/cube/test_kylin_cube_with_slr_ready.json | 2 +-
.../cube/test_kylin_cube_with_slr_ready_2_segments.json | 2 +-
.../localmeta/cube/test_kylin_cube_without_slr_empty.json | 2 +-
.../cube/test_kylin_cube_without_slr_left_join_empty.json | 2 +-
.../cube/test_kylin_cube_without_slr_left_join_ready.json | 2 +-
...st_kylin_cube_without_slr_left_join_ready_2_segments.json | 2 +-
.../localmeta/cube/test_kylin_cube_without_slr_ready.json | 2 +-
.../localmeta/cube/test_streaming_table_cube.json | 2 +-
.../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +-
.../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +-
.../cube_desc/test_kylin_cube_without_slr_desc.json | 2 +-
.../test_kylin_cube_without_slr_left_join_desc.json | 2 +-
.../localmeta/cube_desc/test_streaming_table_cube_desc.json | 2 +-
.../localmeta/hybrid/test_kylin_hybrid_inner_join.json | 2 +-
.../localmeta/hybrid/test_kylin_hybrid_left_join.json | 2 +-
.../localmeta/hybrid/test_kylin_hybrid_ready.json | 2 +-
.../localmeta/invertedindex/test_kylin_ii_inner_join.json | 2 +-
.../localmeta/invertedindex/test_kylin_ii_left_join.json | 2 +-
.../localmeta/invertedindex/test_streaming_table_ii.json | 2 +-
.../invertedindex_desc/test_kylin_ii_inner_join_desc.json | 2 +-
.../invertedindex_desc/test_kylin_ii_left_join_desc.json | 2 +-
.../invertedindex_desc/test_streaming_table_ii_desc.json | 2 +-
.../localmeta/kafka/default.streaming_table.json | 2 +-
.../model_desc/test_kylin_inner_join_model_desc.json | 2 +-
.../model_desc/test_kylin_left_join_model_desc.json | 2 +-
.../model_desc/test_streaming_table_model_desc.json | 2 +-
examples/test_case_data/localmeta/project/default.json | 2 +-
.../localmeta/table/DEFAULT.STREAMING_TABLE.json | 2 +-
.../localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 2 +-
.../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json | 2 +-
examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json | 2 +-
.../localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json | 2 +-
examples/test_case_data/localmeta/table/EDW.TEST_SITES.json | 2 +-
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml | 2 +-
query/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
66 files changed, 66 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 72f2204..16b647e 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 9c296d0..ba5ab1a 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index bfdc662..133b6ff 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 42cf237..74faa49 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -16,16 +16,12 @@
*/
package org.apache.kylin.common;
-/**
- * Current version is: 2.1
- *
- * @since 2.1
- */
+
public class KylinVersion {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final String CURRENT_KYLIN_VERSION = "2.1";
+ private static final String CURRENT_KYLIN_VERSION = "1.5.0";
/**
* Get current Kylin version
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index c46abe7..a82de15 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -76,8 +76,6 @@ abstract public class RootPersistentEntity implements AclEntity, Serializable {
* Metadata model version
*
* User info only, we don't do version control
- *
- * For example: 2.1
*/
@JsonProperty("version")
protected String version = KylinVersion.getCurrentVersion();
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 3757a8b..9fde78a 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 4add5e8..9718b67 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 32f149e..ffe09b0 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index fc89f2f..caf5246 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 2f6fbbb..9bb78ac 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index eb20279..0d0f7b8 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 3ab968f..1fdbded 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index ddd92c6..ab58c59 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube/kylin_sales_cube.json b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
index 7454019..86042ba 100644
--- a/examples/sample_cube/metadata/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
@@ -1,6 +1,6 @@
{
"uuid" : "2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "version":"2.1",
+
"last_modified" : 0,
"name" : "kylin_sales_cube",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
index f0d9fca..963ec9e 100644
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
@@ -1,6 +1,6 @@
{
"uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "version":"2.1",
+
"name" : "kylin_sales_cube_desc",
"description" : null,
"dimensions" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json b/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
index 24ed62d..4cf3c27 100644
--- a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
@@ -1,6 +1,6 @@
{
"uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "version":"2.1",
+
"name" : "kylin_sales_model",
"lookups" : [ {
"table" : "DEFAULT.KYLIN_CAL_DT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/project/learn_kylin.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/project/learn_kylin.json b/examples/sample_cube/metadata/project/learn_kylin.json
index a779954..a4cb1a2 100644
--- a/examples/sample_cube/metadata/project/learn_kylin.json
+++ b/examples/sample_cube/metadata/project/learn_kylin.json
@@ -1,6 +1,6 @@
{
"uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
- "version":"2.1",
+
"name": "learn_kylin",
"realizations": [
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
index 8666feb..225d476 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
@@ -1,6 +1,6 @@
{
"uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "version":"2.1",
+
"name" : "KYLIN_CAL_DT",
"columns" : [ {
"id" : "1",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index e397932..b9891dc 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -1,6 +1,6 @@
{
"uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
- "version":"2.1",
+
"name" : "KYLIN_CATEGORY_GROUPINGS",
"columns" : [ {
"id" : "1",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
index c4a0fb8..4927343 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
@@ -1,6 +1,6 @@
{
"uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
- "version":"2.1",
+
"name" : "KYLIN_SALES",
"columns" : [ {
"id" : "1",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
index 2851b16..7bb8078 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
@@ -1,6 +1,6 @@
{
"uuid" : "111ca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "version":"2.1",
+
"last_modified" : 1404098141020,
"name" : "test_kylin_cube_with_slr_1_new_segment",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
index 71167bd..19c625c 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
@@ -1,6 +1,6 @@
{
"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "version":"2.1",
+
"last_modified" : 0,
"name" : "test_kylin_cube_with_slr_empty",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
index e20fef8..8c4f990 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
@@ -1,6 +1,6 @@
{
"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8dddd",
- "version":"2.1",
+
"last_modified" : 0,
"name" : "test_kylin_cube_with_slr_left_join_empty",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
index 5edaeff..d0aee0b 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
@@ -1,6 +1,6 @@
{
"uuid" : "kkkka32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "version":"2.1",
+
"last_modified" : 1404097095621,
"name" : "test_kylin_cube_with_slr_left_join_ready",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
index bcbf538..568c344 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
@@ -1,6 +1,6 @@
{
"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "version":"2.1",
+
"last_modified" : 1404098141020,
"name" : "test_kylin_cube_with_slr_ready",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
index ee679f5..49c6b74 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
@@ -1,6 +1,6 @@
{
"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53c",
- "version":"2.1",
+
"last_modified" : 1404098141020,
"name" : "test_kylin_cube_with_slr_ready_2_segments",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
index e971c49..c7cfd36 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
@@ -1,6 +1,6 @@
{
"uuid" : "daa53e80-41be-49a5-90ca-9fb7294db186",
- "version":"2.1",
+
"last_modified" : 0,
"name" : "test_kylin_cube_without_slr_empty",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
index 38a9239..672ec2f 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
@@ -1,6 +1,6 @@
{
"uuid" : "dddd3e80-41be-49a5-90ca-9fb7294db186",
- "version":"2.1",
+
"last_modified" : 0,
"name" : "test_kylin_cube_without_slr_left_join_empty",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
index 7a8628a..cf04394 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
@@ -1,6 +1,6 @@
{
"uuid" : "mmmm3e80-41be-49a5-90ca-9fb7294db186",
- "version":"2.1",
+
"last_modified" : 1404097288087,
"name" : "test_kylin_cube_without_slr_left_join_ready",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
index bd26945..8d5b103 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
@@ -1,6 +1,6 @@
{
"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c5ff",
- "version":"2.1",
+
"last_modified" : 1404098141020,
"name" : "test_kylin_cube_without_slr_left_join_ready_2_segments",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
index 90bea1b..6d2f6eb 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
@@ -1,6 +1,6 @@
{
"uuid" : "daa53e80-41be-49a5-90ca-9fb7294db186",
- "version":"2.1",
+
"last_modified" : 1404098303976,
"name" : "test_kylin_cube_without_slr_ready",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube/test_streaming_table_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_streaming_table_cube.json b/examples/test_case_data/localmeta/cube/test_streaming_table_cube.json
index a47513d..7cec5b5 100644
--- a/examples/test_case_data/localmeta/cube/test_streaming_table_cube.json
+++ b/examples/test_case_data/localmeta/cube/test_streaming_table_cube.json
@@ -1,6 +1,6 @@
{
"uuid": "40a27d9d-c5f3-45c4-9b8b-503552419193",
- "version":"2.1",
+
"name": "test_streaming_table_cube",
"owner": "ADMIN",
"cost": 50,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index e164e3e..536f119 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -1,6 +1,6 @@
{
"uuid" : "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
- "version":"2.1",
+
"name" : "test_kylin_cube_with_slr_desc",
"description" : null,
"dimensions" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index 5e450da..c0e39e7 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -1,6 +1,6 @@
{
"uuid" : "bbbba905-1fc6-4f67-985c-38fa5aeafd92",
- "version":"2.1",
+
"name" : "test_kylin_cube_with_slr_left_join_desc",
"description" : null,
"dimensions" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index fbe00d7..d2bcbd2 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -1,6 +1,6 @@
{
- "version":"2.1",
+
"uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
"name": "test_kylin_cube_without_slr_desc",
"description": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index f7a30e0..967c744 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -1,6 +1,6 @@
{
"uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "version":"2.1",
+
"name" : "test_kylin_cube_without_slr_left_join_desc",
"description" : null,
"dimensions" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index 2b26eeb..0267db5 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -1,6 +1,6 @@
{
"uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd192",
- "version":"2.1",
+
"name" : "test_streaming_table_cube_desc",
"description" : "",
"dimensions" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
index 0bdefe8..443c0a3 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
@@ -1,6 +1,6 @@
{
"uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
- "version":"2.1",
+
"name": "test_kylin_hybrid_inner_join",
"realizations": [
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
index 354863c..3a18f87 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
@@ -1,6 +1,6 @@
{
"uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
- "version":"2.1",
+
"name": "test_kylin_hybrid_left_join",
"realizations": [
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
index 1d131c1..8f91a45 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
@@ -1,6 +1,6 @@
{
"uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
- "version":"2.1",
+
"name": "test_kylin_hybrid_ready",
"realizations": [
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
index 5b8614f..13fc282 100644
--- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
+++ b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
@@ -1,5 +1,5 @@
{
- "model_version":"2.1",
+
"uuid": "dk853e80-41be-49a5-89ca-9fb7294db186",
"name": "test_kylin_ii_inner_join",
"owner": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
index 67b4bca..28b23ef 100644
--- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
+++ b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "daa53e80-41be-49a5-89ca-9fb7294db186",
"name": "test_kylin_ii_left_join",
"owner": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
index 689f04c..4ec2711 100644
--- a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
+++ b/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "daa53e80-41be-49a5-89ca-9fb729999812",
"name": "test_streaming_table_ii",
"owner": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
index 71549c1..eb383fb 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "88bf87b5-c7b5-4420-a12a-07f6b37b3187",
"last_modified": 0,
"name": "test_kylin_ii_inner_join_desc",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
index a630490..2f152ad 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "74bf87b5-c7b5-4420-a12a-07f6b37b3187",
"last_modified": 0,
"name": "test_kylin_ii_left_join_desc",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
index e5e6202..c39ae90 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "74bf87b5-c7b5-4420-a12a-07f6b3864789",
"last_modified": 0,
"name": "test_streaming_table_ii_desc",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/kafka/default.streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kafka/default.streaming_table.json b/examples/test_case_data/localmeta/kafka/default.streaming_table.json
index c99b8e5..c97927d 100644
--- a/examples/test_case_data/localmeta/kafka/default.streaming_table.json
+++ b/examples/test_case_data/localmeta/kafka/default.streaming_table.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
"name": "default.streaming_table",
"topic": "test_streaming_table_topic_xyz",
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index d908827..c40a9e1 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
"name" : "test_kylin_inner_join_model_desc",
"lookups" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index cc413b2..1e8a6e9 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_left_join_model_desc",
"lookups": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
index e7392d6..f80d35f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "ff527b94-f860-44c3-8452-93b177888732",
"name": "test_streaming_table_model_desc",
"dimensions": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 8d138dd..6359ff5 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
"name": "default",
"realizations": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
index 6030a5e..6ead441 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid": "e286e39e-40d7-44c2-8fa2-41b365123987",
"name": "STREAMING_TABLE",
"columns": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
index 6929203..7d2964f 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
"name" : "TEST_CATEGORY_GROUPINGS",
"columns" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 2e227d6..ee21ea0 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
"name" : "TEST_KYLIN_FACT",
"columns" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
index 7d78673..9296109 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
"name" : "TEST_CAL_DT",
"columns" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
index 6bfd3c4..6bc2815 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1",
"name" : "TEST_SELLER_TYPE_DIM",
"columns" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
index cab1edb..d5cc116 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
@@ -1,5 +1,5 @@
{
- "version":"2.1",
+
"uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6",
"name" : "TEST_SITES",
"columns" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 6d75dc0..5007629 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 3867385..dae9d1e 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 99b650c..e3fbc84 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 501bf77..9caa29c3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<name>Kylin:HadoopOLAPEngine</name>
<url>http://kylin.apache.org</url>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 2d1bea9..55add61 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 7c1d58a..b9ea034 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 9adc02b..46c016d 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 9093dea..7a8d7b0 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3569b22b/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 965eb53..c274874 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
[28/50] [abbrv] kylin git commit: fix bug and strength UT in cuboid
scheduling
Posted by sh...@apache.org.
fix bug and strength UT in cuboid scheduling
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea4a42c6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea4a42c6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea4a42c6
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: ea4a42c63af0669c0b964eb7bb20ca7dcc52fa56
Parents: 8590b2d
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 10 19:46:56 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 10 19:47:02 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/cuboid/Cuboid.java | 1 -
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 32 ++++++++++++++++++++
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 10 ++++++
3 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea4a42c6/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index c4e373b..16b0287 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -235,7 +235,6 @@ public class Cuboid implements Comparable<Cuboid> {
for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
long result = cuboidID & hierarchyMasks.fullMask;
if (result > 0) {
- // if match one of the hierarchy constrains, return true;
for (long mask : hierarchyMasks.allMasks) {
if (result == mask) {
continue hier;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea4a42c6/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index 49382bd..d5918aa 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -57,7 +57,9 @@ public class CuboidCLI {
if (!notfound) {
throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
}
+
cuboidQueue.push(sc);
+
}
}
@@ -68,6 +70,36 @@ public class CuboidCLI {
if (enumCuboids.equals(cuboidSet) == false) {
throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
}
+
+ //check all valid and invalid
+ for (long i = 0; i < baseCuboid; ++i) {
+ if (cuboidSet.contains(i)) {
+ if (!Cuboid.isValid(cubeDesc, i)) {
+ throw new RuntimeException();
+ }
+
+ if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
+ throw new RuntimeException();
+ }
+ } else {
+ if (Cuboid.isValid(cubeDesc, i)) {
+ throw new RuntimeException();
+ }
+
+ long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
+ if (corrected == i) {
+ throw new RuntimeException();
+ }
+
+ if (!Cuboid.isValid(cubeDesc, corrected)) {
+ throw new RuntimeException();
+ }
+
+ if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
+ throw new RuntimeException();
+ }
+ }
+ }
}
return cuboidSet.size();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea4a42c6/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 0711d6e..02c01b8 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -88,6 +88,10 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
private CubeDesc getStreamingCubeDesc() {
return getCubeDescManager().getCubeDesc("test_streaming_table_cube_desc");
}
+
+ private CubeDesc getSSBCubeDesc() {
+ return getCubeDescManager().getCubeDesc("ssb");
+ }
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
for (long cuboidId : cuboidIds) {
@@ -209,6 +213,12 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
}
@Test
+ public void testCuboidGeneration6() {
+ CubeDesc cube = getSSBCubeDesc();
+ CuboidCLI.simulateCuboidGeneration(cube, true);
+ }
+
+ @Test
public void testCuboidCounts1() {
CubeDesc cube = getTestKylinCubeWithoutSeller();
CuboidScheduler cuboidScheduler = new CuboidScheduler(cube);
[17/50] [abbrv] kylin git commit: KYLIN-1421 fix the “Last build time” is always empty issue
Posted by sh...@apache.org.
KYLIN-1421 fix the “Last build time” is always empty issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ec3ed0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ec3ed0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ec3ed0e
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 0ec3ed0e899eeab1497667ed28f10226aec520e7
Parents: d1a574b
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 14:07:00 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:21:56 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ec3ed0e/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index b4182fe..f5cb66e 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -118,6 +118,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
CubeSegment segment = cubeManager.appendSegments(cubeInstance, streamingBatch.getTimeRange().getFirst(), streamingBatch.getTimeRange().getSecond(), false, false);
segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
segment.setInputRecords(streamingBatch.getMessages().size());
+ segment.setLastBuildTime(System.currentTimeMillis());
return segment;
} catch (IOException e) {
throw new RuntimeException("failed to create IBuildable", e);
[18/50] [abbrv] kylin git commit: initial commit for KYLIN-1431
Posted by sh...@apache.org.
initial commit for KYLIN-1431
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9021f17d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9021f17d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9021f17d
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 9021f17d85be01bf34b48a7a31be82f53ceb9c8f
Parents: 0ec3ed0
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 2 11:16:46 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 15:07:04 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/CubeMetaExtractor.java | 2 +-
build/bin/streaming_build.sh | 4 +-
build/bin/streaming_fillgap.sh | 5 +-
.../kylin/metadata/model/ISourceAware.java | 1 +
.../kylin/engine/streaming/BootstrapConfig.java | 20 +-
.../kylin/engine/streaming/IStreamingInput.java | 3 +-
.../streaming/OneOffStreamingBuilder.java | 17 +-
.../kylin/engine/streaming/StreamingConfig.java | 33 +--
.../engine/streaming/StreamingManager.java | 12 +
.../engine/streaming/cli/StreamingCLI.java | 21 +-
.../engine/streaming/util/StreamingUtils.java | 18 +-
.../kafka/default.streaming_table.json | 21 ++
.../localmeta/kafka/kafka_test.json | 20 --
.../kafka/test_streaming_table_cube.json | 22 --
.../kafka/test_streaming_table_ii.json | 22 --
.../streaming/default.streaming_table.json | 6 +
.../localmeta/streaming/kafka_test.json | 20 --
.../streaming/test_streaming_table_cube.json | 8 -
.../streaming/test_streaming_table_ii.json | 8 -
.../kylin/provision/BuildCubeWithStream.java | 16 +-
.../kylin/rest/controller/CubeController.java | 234 -------------------
.../rest/controller/StreamingController.java | 4 +-
.../kylin/rest/service/StreamingService.java | 18 +-
.../kylin/source/kafka/KafkaStreamingInput.java | 78 ++++---
.../kylin/source/kafka/StreamingParser.java | 6 +-
25 files changed, 163 insertions(+), 456 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java b/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
index 527ef0a..ef27ade 100644
--- a/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
+++ b/assembly/src/test/java/org/apache/kylin/job/CubeMetaExtractor.java
@@ -227,7 +227,7 @@ public class CubeMetaExtractor extends AbstractApplication {
private void dealWithStreaming(CubeInstance cube) {
for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
- if (streamingConfig.getCubeName() != null && streamingConfig.getCubeName().equalsIgnoreCase(cube.getName())) {
+ if (streamingConfig.getName() != null && streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) {
requiredResources.add(StreamingConfig.concatResourcePath(streamingConfig.getName()));
requiredResources.add(KafkaConfig.concatResourcePath(streamingConfig.getName()));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/build/bin/streaming_build.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_build.sh b/build/bin/streaming_build.sh
index a96ecc1..ed19036 100644
--- a/build/bin/streaming_build.sh
+++ b/build/bin/streaming_build.sh
@@ -20,7 +20,7 @@
source /etc/profile
source ~/.bash_profile
-STREAMING=$1
+CUBE=$1
INTERVAL=$2
DELAY=$3
CURRENT_TIME_IN_SECOND=`date +%s`
@@ -30,4 +30,4 @@ END=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY + INTERVAL))
ID="$START"_"$END"
echo "building for ${ID}" >> ${KYLIN_HOME}/logs/build_trace.log
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${STREAMING} ${ID} -start ${START} -end ${END} -streaming ${STREAMING}
\ No newline at end of file
+sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${CUBE} ${ID} -start ${START} -end ${END} -cube ${CUBE}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/build/bin/streaming_fillgap.sh
----------------------------------------------------------------------
diff --git a/build/bin/streaming_fillgap.sh b/build/bin/streaming_fillgap.sh
index 74d9037..c67809a 100644
--- a/build/bin/streaming_fillgap.sh
+++ b/build/bin/streaming_fillgap.sh
@@ -20,8 +20,7 @@
source /etc/profile
source ~/.bash_profile
-streaming=$1
-margin=$2
+cube=$1
cd ${KYLIN_HOME}
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${streaming} fillgap -streaming ${streaming} -fillGap true -margin ${margin}
\ No newline at end of file
+sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${cube} fillgap -cube ${cube} -fillGap true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java
index 3d89f40..8cfda15 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
public interface ISourceAware {
public static final int ID_HIVE = 0;
+ public static final int ID_STREAMING = 1;
public static final int ID_SPARKSQL = 5;
int getSourceType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
index a3e2db5..a4c4618 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
@@ -4,9 +4,7 @@ package org.apache.kylin.engine.streaming;
*/
public class BootstrapConfig {
- private String streaming;
- private int partitionId = -1;
-
+ private String cubeName;
private long start = 0L;
private long end = 0L;
@@ -28,20 +26,12 @@ public class BootstrapConfig {
this.end = end;
}
- public String getStreaming() {
- return streaming;
- }
-
- public void setStreaming(String streaming) {
- this.streaming = streaming;
- }
-
- public int getPartitionId() {
- return partitionId;
+ public String getCubeName() {
+ return cubeName;
}
- public void setPartitionId(int partitionId) {
- this.partitionId = partitionId;
+ public void setCubeName(String cubeName) {
+ this.cubeName = cubeName;
}
public boolean isFillGap() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
index 1cf3d98..4b4cf02 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/IStreamingInput.java
@@ -34,11 +34,12 @@
package org.apache.kylin.engine.streaming;
import org.apache.kylin.common.util.StreamingBatch;
+import org.apache.kylin.metadata.realization.RealizationType;
/**
*/
public interface IStreamingInput {
- StreamingBatch getBatchWithTimeWindow(String streamingConfig, int id, long startTime, long endTime);
+ StreamingBatch getBatchWithTimeWindow(RealizationType realizationType, String realizationName, int id, long startTime, long endTime);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
index 3fbade2..6bad000 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
@@ -43,6 +43,7 @@ import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
+import org.apache.kylin.metadata.realization.RealizationType;
/**
*/
@@ -53,23 +54,25 @@ public class OneOffStreamingBuilder {
private final StreamingBatchBuilder streamingBatchBuilder;
private final long startTime;
private final long endTime;
- private final String streamingConfig;
+ private final RealizationType realizationType;
+ private final String realizationName;
- public OneOffStreamingBuilder(String streamingConfig, long startTime, long endTime) {
+ public OneOffStreamingBuilder(RealizationType realizationType, String realizationName, long startTime, long endTime) {
Preconditions.checkArgument(startTime < endTime);
this.startTime = startTime;
this.endTime = endTime;
- this.streamingConfig = Preconditions.checkNotNull(streamingConfig);
- this.streamingInput = Preconditions.checkNotNull(StreamingUtils.getStreamingInput(streamingConfig));
- this.streamingOutput = Preconditions.checkNotNull(StreamingUtils.getStreamingOutput(streamingConfig));
- this.streamingBatchBuilder = Preconditions.checkNotNull(StreamingUtils.getMicroBatchBuilder(streamingConfig));
+ this.realizationType = Preconditions.checkNotNull(realizationType);
+ this.realizationName = Preconditions.checkNotNull(realizationName);
+ this.streamingInput = Preconditions.checkNotNull(StreamingUtils.getStreamingInput());
+ this.streamingOutput = Preconditions.checkNotNull(StreamingUtils.getStreamingOutput());
+ this.streamingBatchBuilder = Preconditions.checkNotNull(StreamingUtils.getMicroBatchBuilder(realizationType, realizationName));
}
public Runnable build() {
return new Runnable() {
@Override
public void run() {
- StreamingBatch streamingBatch = streamingInput.getBatchWithTimeWindow(streamingConfig, -1, startTime, endTime);
+ StreamingBatch streamingBatch = streamingInput.getBatchWithTimeWindow(realizationType, realizationName, -1, startTime, endTime);
final IBuildable buildable = streamingBatchBuilder.createBuildable(streamingBatch);
final Map<Long, HyperLogLogPlusCounter> samplingResult = streamingBatchBuilder.sampling(streamingBatch);
final Map<TblColRef, Dictionary<String>> dictionaryMap = streamingBatchBuilder.buildDictionary(streamingBatch, buildable);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
index f0a7ab1..c8d1911 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingConfig.java
@@ -55,29 +55,24 @@ public class StreamingConfig extends RootPersistentEntity {
public static Serializer<StreamingConfig> SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
+ public static final String STREAMING_TYPE_KAFKA = "kafka";
+
@JsonProperty("name")
private String name;
- @JsonProperty("iiName")
- private String iiName;
-
- @JsonProperty("cubeName")
- private String cubeName;
+ @JsonProperty("type")
+ private String type = STREAMING_TYPE_KAFKA;
- public String getCubeName() {
- return cubeName;
+ public String getType() {
+ return type;
}
- public void setCubeName(String cubeName) {
- this.cubeName = cubeName;
+ public void setType(String type) {
+ this.type = type;
}
- public String getIiName() {
- return iiName;
- }
-
- public void setIiName(String iiName) {
- this.iiName = iiName;
+ public String getResourcePath() {
+ return concatResourcePath(name);
}
public String getName() {
@@ -88,12 +83,8 @@ public class StreamingConfig extends RootPersistentEntity {
this.name = name;
}
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String streamingName) {
- return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + streamingName + ".json";
+ public static String concatResourcePath(String name) {
+ return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
index e0b086d..f652762 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java
@@ -104,6 +104,18 @@ public class StreamingManager {
}
}
+ private static String formatStreamingConfigPath(String name) {
+ return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json";
+ }
+
+ private static String formatStreamingOutputPath(String streaming, int partition) {
+ return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + partition + ".json";
+ }
+
+ private static String formatStreamingOutputPath(String streaming, List<Integer> partitions) {
+ return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + StringUtils.join(partitions, "_") + ".json";
+ }
+
public StreamingConfig getStreamingConfig(String name) {
return streamingMap.get(name);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
index a73a6ac..0bab396 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
@@ -44,6 +44,7 @@ import org.apache.kylin.engine.streaming.OneOffStreamingBuilder;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.engine.streaming.monitor.StreamingMonitor;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,7 +56,7 @@ public class StreamingCLI {
public static void main(String[] args) {
try {
- Preconditions.checkArgument(args[0].equals("streaming"));
+ Preconditions.checkArgument(args[0].equals("cube"));
Preconditions.checkArgument(args[1].equals("start"));
int i = 2;
@@ -69,11 +70,8 @@ public class StreamingCLI {
case "-end":
bootstrapConfig.setEnd(Long.parseLong(args[++i]));
break;
- case "-streaming":
- bootstrapConfig.setStreaming(args[++i]);
- break;
- case "-partition":
- bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
+ case "-cube":
+ bootstrapConfig.setCubeName(args[++i]);
break;
case "-fillGap":
bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
@@ -84,14 +82,13 @@ public class StreamingCLI {
i++;
}
if (bootstrapConfig.isFillGap()) {
- final StreamingConfig streamingConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getStreamingConfig(bootstrapConfig.getStreaming());
- final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(streamingConfig.getCubeName());
+ final List<Pair<Long, Long>> gaps = StreamingMonitor.findGaps(bootstrapConfig.getCubeName());
logger.info("all gaps:" + StringUtils.join(gaps, ","));
for (Pair<Long, Long> gap : gaps) {
- startOneOffCubeStreaming(bootstrapConfig.getStreaming(), gap.getFirst(), gap.getSecond());
+ startOneOffCubeStreaming(bootstrapConfig.getCubeName(), gap.getFirst(), gap.getSecond());
}
} else {
- startOneOffCubeStreaming(bootstrapConfig.getStreaming(), bootstrapConfig.getStart(), bootstrapConfig.getEnd());
+ startOneOffCubeStreaming(bootstrapConfig.getCubeName(), bootstrapConfig.getStart(), bootstrapConfig.getEnd());
logger.info("streaming process finished, exit with 0");
System.exit(0);
}
@@ -102,8 +99,8 @@ public class StreamingCLI {
}
}
- private static void startOneOffCubeStreaming(String streaming, long start, long end) {
- final Runnable runnable = new OneOffStreamingBuilder(streaming, start, end).build();
+ private static void startOneOffCubeStreaming(String cubeName, long start, long end) {
+ final Runnable runnable = new OneOffStreamingBuilder(RealizationType.CUBE, cubeName, start, end).build();
runnable.run();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
index 0ae7143..66a0af2 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
@@ -43,29 +43,27 @@ import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.engine.streaming.cube.StreamingCubeBuilder;
import com.google.common.base.Preconditions;
+import org.apache.kylin.metadata.realization.RealizationType;
/**
* TODO: like MRUtil, use Factory pattern to allow config
*/
public class StreamingUtils {
- public static IStreamingInput getStreamingInput(String streaming) {
+ public static IStreamingInput getStreamingInput() {
return (IStreamingInput) ClassUtil.newInstance("org.apache.kylin.source.kafka.KafkaStreamingInput");
}
- public static IStreamingOutput getStreamingOutput(String streaming) {
+ public static IStreamingOutput getStreamingOutput() {
return (IStreamingOutput) ClassUtil.newInstance("org.apache.kylin.storage.hbase.steps.HBaseStreamingOutput");
}
- public static StreamingBatchBuilder getMicroBatchBuilder(String streaming) {
- final StreamingConfig streamingConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getStreamingConfig(streaming);
- Preconditions.checkNotNull(streamingConfig);
- if (streamingConfig.getCubeName() != null) {
- return new StreamingCubeBuilder(streamingConfig.getCubeName());
- } else if (streamingConfig.getIiName() != null) {
- throw new UnsupportedOperationException("not implemented yet");
+ public static StreamingBatchBuilder getMicroBatchBuilder(RealizationType realizationType, String realizationName) {
+ Preconditions.checkNotNull(realizationName);
+ if (realizationType == RealizationType.CUBE) {
+ return new StreamingCubeBuilder(realizationName);
} else {
- throw new UnsupportedOperationException("StreamingConfig is not valid");
+ throw new UnsupportedOperationException("not implemented yet");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/kafka/default.streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kafka/default.streaming_table.json b/examples/test_case_data/localmeta/kafka/default.streaming_table.json
new file mode 100644
index 0000000..c99b8e5
--- /dev/null
+++ b/examples/test_case_data/localmeta/kafka/default.streaming_table.json
@@ -0,0 +1,21 @@
+{
+ "version":"2.1",
+ "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+ "name": "default.streaming_table",
+ "topic": "test_streaming_table_topic_xyz",
+ "timeout": 60000,
+ "bufferSize": 65536,
+ "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
+ "last_modified": 0,
+ "clusters": [
+ {
+ "brokers": [
+ {
+ "id": 0,
+ "host": "sandbox",
+ "port": 6667
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/kafka/kafka_test.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kafka/kafka_test.json b/examples/test_case_data/localmeta/kafka/kafka_test.json
deleted file mode 100644
index a20f71e..0000000
--- a/examples/test_case_data/localmeta/kafka/kafka_test.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec6c3c0d",
- "name": "kafka_test",
- "topic": "kafka_stream_test",
- "timeout": 60000,
- "bufferSize": 65536,
- "last_modified": 0,
- "clusters": [
- {
- "brokers": [
- {
- "id": 0,
- "host": "sandbox.hortonworks.com",
- "port": 6667
- }
- ]
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/kafka/test_streaming_table_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kafka/test_streaming_table_cube.json b/examples/test_case_data/localmeta/kafka/test_streaming_table_cube.json
deleted file mode 100644
index 554fa62..0000000
--- a/examples/test_case_data/localmeta/kafka/test_streaming_table_cube.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
- "name": "test_streaming_table_cube",
- "topic": "test_streaming_table_topic_xyz",
- "timeout": 60000,
- "bufferSize": 65536,
- "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
- "partition": 1,
- "last_modified": 0,
- "clusters": [
- {
- "brokers": [
- {
- "id": 0,
- "host": "sandbox",
- "port": 6667
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/kafka/test_streaming_table_ii.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kafka/test_streaming_table_ii.json b/examples/test_case_data/localmeta/kafka/test_streaming_table_ii.json
deleted file mode 100644
index b6f18c7..0000000
--- a/examples/test_case_data/localmeta/kafka/test_streaming_table_ii.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec909322",
- "name": "test_streaming_table_ii",
- "topic": "test_streaming_table_topic_xyz",
- "timeout": 60000,
- "bufferSize": 65536,
- "parserName": "org.apache.kylin.source.kafka.JsonStreamParser",
- "partition": 1,
- "last_modified": 0,
- "clusters": [
- {
- "brokers": [
- {
- "id": 0,
- "host": "sandbox",
- "port": 6667
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/streaming/default.streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/default.streaming_table.json b/examples/test_case_data/localmeta/streaming/default.streaming_table.json
new file mode 100644
index 0000000..6eb4a88
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/default.streaming_table.json
@@ -0,0 +1,6 @@
+{
+ "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+ "name": "default.streaming_table",
+ "type": "kafka",
+ "last_modified": 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/streaming/kafka_test.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/kafka_test.json b/examples/test_case_data/localmeta/streaming/kafka_test.json
deleted file mode 100644
index a20f71e..0000000
--- a/examples/test_case_data/localmeta/streaming/kafka_test.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec6c3c0d",
- "name": "kafka_test",
- "topic": "kafka_stream_test",
- "timeout": 60000,
- "bufferSize": 65536,
- "last_modified": 0,
- "clusters": [
- {
- "brokers": [
- {
- "id": 0,
- "host": "sandbox.hortonworks.com",
- "port": 6667
- }
- ]
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/streaming/test_streaming_table_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/test_streaming_table_cube.json b/examples/test_case_data/localmeta/streaming/test_streaming_table_cube.json
deleted file mode 100644
index ecf0511..0000000
--- a/examples/test_case_data/localmeta/streaming/test_streaming_table_cube.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
- "name": "test_streaming_table_cube",
- "cubeName": "test_streaming_table_cube",
- "partition": 1,
- "last_modified": 0
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/examples/test_case_data/localmeta/streaming/test_streaming_table_ii.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/test_streaming_table_ii.json b/examples/test_case_data/localmeta/streaming/test_streaming_table_ii.json
deleted file mode 100644
index 022ab70..0000000
--- a/examples/test_case_data/localmeta/streaming/test_streaming_table_ii.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "version":"2.1",
- "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec909322",
- "name": "test_streaming_table_ii",
- "iiName": "test_streaming_table_ii",
- "partition": 1,
- "last_modified": 0
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 27226e7..eeff999 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -27,10 +27,13 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.streaming.OneOffStreamingBuilder;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.job.DeployUtil;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.source.kafka.KafkaConfigManager;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
@@ -42,7 +45,7 @@ import org.slf4j.LoggerFactory;
public class BuildCubeWithStream {
private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStream.class);
- private static final String streamingName = "test_streaming_table_cube";
+ private static final String cubeName = "test_streaming_table_cube";
private static final long startTime = DateFormat.stringToMillis("2015-01-01 00:00:00");
private static final long endTime = DateFormat.stringToMillis("2015-01-03 00:00:00");
private static final long batchInterval = 16 * 60 * 60 * 1000;//16 hours
@@ -75,15 +78,16 @@ public class BuildCubeWithStream {
DeployUtil.overrideJobJarLocations();
kylinConfig = KylinConfig.getInstanceFromEnv();
-
- final StreamingConfig config = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getStreamingConfig(streamingName);
+ final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
+ final String factTable = cubeInstance.getFactTable();
+ final StreamingConfig config = StreamingManager.getInstance(kylinConfig).getStreamingConfig(factTable);
//Use a random topic for kafka data stream
- KafkaConfig streamingConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingName);
+ KafkaConfig streamingConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(config.getName());
streamingConfig.setTopic(UUID.randomUUID().toString());
KafkaConfigManager.getInstance(kylinConfig).saveKafkaConfig(streamingConfig);
- DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, config.getCubeName(), streamingConfig);
+ DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, cubeName, streamingConfig);
}
public static void afterClass() throws Exception {
@@ -94,7 +98,7 @@ public class BuildCubeWithStream {
logger.info("start time:" + startTime + " end time:" + endTime + " batch interval:" + batchInterval + " batch count:" + ((endTime - startTime) / batchInterval));
for (long start = startTime; start < endTime; start += batchInterval) {
logger.info(String.format("build batch:{%d, %d}", start, start + batchInterval));
- new OneOffStreamingBuilder(streamingName, start, start + batchInterval).build().run();
+ new OneOffStreamingBuilder(RealizationType.CUBE, cubeName, start, start + batchInterval).build().run();
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 9afa750..e60f330 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -284,92 +284,6 @@ public class CubeController extends BasicController {
throw new InternalErrorException("Failed to clone cube ", e);
}
- boolean isStreamingCube = false, cloneStreamingConfigSuccess = false, cloneKafkaConfigSuccess = false;
-
-
- List<StreamingConfig> streamingConfigs = null;
- try {
- streamingConfigs = streamingService.listAllStreamingConfigs(cubeName);
- if (streamingConfigs.size() != 0) {
- isStreamingCube = true;
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- StreamingConfig newStreamingConfig = null;
- KafkaConfig newKafkaConfig = null;
-
- try {
-
- if (isStreamingCube) {
-
- isStreamingCube = true;
- newStreamingConfig = streamingConfigs.get(0).clone();
- newStreamingConfig.setName(newCubeName + "_STREAMING");
- newStreamingConfig.updateRandomUuid();
- newStreamingConfig.setLastModified(0);
- newStreamingConfig.setCubeName(newCubeName);
- try {
- streamingService.createStreamingConfig(newStreamingConfig);
- cloneStreamingConfigSuccess = true;
- } catch (IOException e) {
- throw new InternalErrorException("Failed to clone streaming config. ", e);
- }
-
- //StreamingConfig name and KafkaConfig name is the same for same cube
- String kafkaConfigName = streamingConfigs.get(0).getName();
- KafkaConfig kafkaConfig = null;
- try {
- kafkaConfig = kafkaConfigService.getKafkaConfig(kafkaConfigName);
- if (kafkaConfig != null) {
- newKafkaConfig = kafkaConfig.clone();
- newKafkaConfig.setName(newStreamingConfig.getName());
- newKafkaConfig.setLastModified(0);
- newKafkaConfig.updateRandomUuid();
- }
- } catch (IOException e) {
- throw new InternalErrorException("Failed to get kafka config info. ", e);
- }
-
- try {
- kafkaConfigService.createKafkaConfig(newKafkaConfig);
- cloneKafkaConfigSuccess = true;
- } catch (IOException e) {
- throw new InternalErrorException("Failed to clone streaming config. ", e);
- }
- }
- } finally {
-
- //rollback if failed
- if (isStreamingCube) {
- if (cloneStreamingConfigSuccess == false || cloneKafkaConfigSuccess == false) {
- try {
- cubeService.deleteCube(newCube);
- } catch (Exception ex) {
- throw new InternalErrorException("Failed, and failed to rollback on delete cube. " + " Caused by: " + ex.getMessage(), ex);
- }
- if (cloneStreamingConfigSuccess == true) {
- try {
- streamingService.dropStreamingConfig(newStreamingConfig);
- } catch (IOException e) {
- throw new InternalErrorException("Failed to clone cube, and StreamingConfig created and failed to delete: " + e.getLocalizedMessage());
- }
- }
- if (cloneKafkaConfigSuccess == true) {
- try {
- kafkaConfigService.dropKafkaConfig(newKafkaConfig);
- } catch (IOException e) {
- throw new InternalErrorException("Failed to clone cube, and KafkaConfig created and failed to delete: " + e.getLocalizedMessage());
- }
- }
-
- }
-
- }
- }
-
return newCube;
}
@@ -400,27 +314,6 @@ public class CubeController extends BasicController {
throw new NotFoundException("Cube with name " + cubeName + " not found..");
}
- //drop related StreamingConfig KafkaConfig if exist
- try {
- List<StreamingConfig> configs = streamingService.listAllStreamingConfigs(cubeName);
- for (StreamingConfig config : configs) {
- try {
- streamingService.dropStreamingConfig(config);
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to delete StreamingConfig. " + " Caused by: " + e.getMessage(), e);
- }
- try {
- KafkaConfig kfkConfig = kafkaConfigService.getKafkaConfig(config.getName());
- kafkaConfigService.dropKafkaConfig(kfkConfig);
- } catch (IOException e) {
- throw new InternalErrorException("Failed to delete KafkaConfig. " + " Caused by: " + e.getMessage(), e);
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
//drop Cube
try {
cubeService.deleteCube(cube);
@@ -587,133 +480,6 @@ public class CubeController extends BasicController {
return cubeRequest;
}
- boolean updateStreamingConfigSuccess = false, updateKafkaConfigSuccess = false;
-
- boolean isStreamingCube = cubeRequest.getStreamingCube() != null && cubeRequest.getStreamingCube().equals("true");
-
- //oldConfig is for recover use
- StreamingConfig streamingConfig = null, oldStreamingConfig = null;
- KafkaConfig kafkaConfig = null, oldKafkaConfig = null;
- if (isStreamingCube) {
- streamingConfig = deserializeStreamingDesc(cubeRequest);
- kafkaConfig = deserializeKafkaDesc(cubeRequest);
- try {
- oldKafkaConfig = kafkaConfigService.getKafkaConfig(kafkaConfig.getName());
- } catch (IOException e) {
- e.printStackTrace();
- }
- oldStreamingConfig = streamingService.getStreamingManager().getStreamingConfig(streamingConfig.getName());
- }
- try {
- //streaming Cube
- if (isStreamingCube) {
- if (streamingConfig == null) {
- cubeRequest.setMessage("No StreamingConfig info to update.");
- return cubeRequest;
- }
- if (kafkaConfig == null) {
- cubeRequest.setMessage("No KafkaConfig info to update.");
- return cubeRequest;
- }
-
- if (oldStreamingConfig == null) {
- streamingConfig.setUuid(UUID.randomUUID().toString());
- try {
- streamingService.createStreamingConfig(streamingConfig);
- updateStreamingConfigSuccess = true;
- } catch (IOException e) {
- logger.error("Failed to add StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to add StreamingConfig: " + e.getLocalizedMessage());
- }
- } else {
- try {
- streamingConfig = streamingService.updateStreamingConfig(streamingConfig);
- updateStreamingConfigSuccess = true;
-
- } catch (IOException e) {
- logger.error("Failed to update StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to update StreamingConfig: " + e.getLocalizedMessage());
- }
- }
- if (oldKafkaConfig == null) {
- kafkaConfig.setUuid(UUID.randomUUID().toString());
- try {
- kafkaConfigService.createKafkaConfig(kafkaConfig);
- updateKafkaConfigSuccess = true;
- } catch (IOException e) {
- logger.error("Failed to add KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to add KafkaConfig: " + e.getLocalizedMessage());
- }
-
- } else {
- try {
- kafkaConfig = kafkaConfigService.updateKafkaConfig(kafkaConfig);
- updateKafkaConfigSuccess = true;
- } catch (IOException e) {
- logger.error("Failed to update KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to update KafkaConfig: " + e.getLocalizedMessage());
- }
- }
-
- }
- } finally {
- if (isStreamingCube) {
- //recover cube desc
- if (updateStreamingConfigSuccess == false || updateKafkaConfigSuccess == false) {
- oldCubeDesc.setLastModified(desc.getLastModified());
- CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
- try {
- desc = cubeService.updateCubeAndDesc(cube, oldCubeDesc, projectName);
- } catch (Exception e) {
- logger.error("Failed to recover CubeDesc:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to recover CubeDesc: " + e.getLocalizedMessage());
- }
-
- if (updateStreamingConfigSuccess == true) {
-
- if (oldStreamingConfig != null) {
-
- oldStreamingConfig.setLastModified(streamingConfig.getLastModified());
- try {
- streamingService.updateStreamingConfig(oldStreamingConfig);
- } catch (IOException e) {
- logger.error("Failed to recover StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to recover StreamingConfig: " + e.getLocalizedMessage());
- }
- } else {
- try {
- streamingService.dropStreamingConfig(streamingConfig);
- } catch (IOException e) {
- logger.error("Failed to remove added StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to remove added StreamingConfig: " + e.getLocalizedMessage());
- }
- }
- }
-
- if (updateKafkaConfigSuccess == true) {
- if (oldKafkaConfig != null) {
- oldKafkaConfig.setLastModified(kafkaConfig.getLastModified());
- try {
- kafkaConfigService.updateKafkaConfig(oldKafkaConfig);
- } catch (IOException e) {
- logger.error("Failed to recover KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to recover KafkaConfig: " + e.getLocalizedMessage());
- }
- } else {
- try {
- kafkaConfigService.dropKafkaConfig(kafkaConfig);
- } catch (IOException e) {
- logger.error("Failed to remove added KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to remove added KafkaConfig: " + e.getLocalizedMessage());
- }
- }
- }
-
- }
- }
-
- }
-
String descData = JsonUtil.writeValueAsIndentString(desc);
cubeRequest.setCubeDescData(descData);
cubeRequest.setSuccessful(true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e22bd30..ecd7571 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -60,9 +60,9 @@ public class StreamingController extends BasicController {
@RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
@ResponseBody
- public List<StreamingConfig> getStreamings(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
+ public List<StreamingConfig> getStreamings(@RequestParam(value = "table", required = false) String table, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
try {
- return streamingService.getStreamingConfigs(cubeName, limit, offset);
+ return streamingService.getStreamingConfigs(table, limit, offset);
} catch (IOException e) {
logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
index e40426b..a0473e9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.rest.constant.Constant;
@@ -37,26 +39,22 @@ public class StreamingService extends BasicService {
private AccessService accessService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<StreamingConfig> listAllStreamingConfigs(final String cubeName) throws IOException {
+ public List<StreamingConfig> listAllStreamingConfigs(final String table) throws IOException {
List<StreamingConfig> streamingConfigs = new ArrayList();
- CubeInstance cubeInstance = (null != cubeName) ? getCubeManager().getCube(cubeName) : null;
- if (null == cubeInstance) {
+ if (StringUtils.isEmpty(table)) {
streamingConfigs = getStreamingManager().listAllStreaming();
} else {
- for(StreamingConfig config : getStreamingManager().listAllStreaming()){
- if(cubeInstance.getName().equals(config.getCubeName())){
- streamingConfigs.add(config);
- }
- }
+ StreamingConfig config = getStreamingManager().getConfig(table);
+ streamingConfigs.add(config);
}
return streamingConfigs;
}
- public List<StreamingConfig> getStreamingConfigs(final String cubeName, final Integer limit, final Integer offset) throws IOException {
+ public List<StreamingConfig> getStreamingConfigs(final String table, final Integer limit, final Integer offset) throws IOException {
List<StreamingConfig> streamingConfigs;
- streamingConfigs = listAllStreamingConfigs(cubeName);
+ streamingConfigs = listAllStreamingConfigs(table);
if (limit == null || offset == null) {
return streamingConfigs;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index 2e262b3..c05119f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -47,9 +47,14 @@ import kafka.message.MessageAndOffset;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.streaming.IStreamingInput;
import org.apache.kylin.common.util.StreamingBatch;
import org.apache.kylin.common.util.StreamingMessage;
+import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.apache.kylin.source.kafka.util.KafkaRequester;
@@ -65,39 +70,54 @@ public class KafkaStreamingInput implements IStreamingInput {
private static final Logger logger = LoggerFactory.getLogger(KafkaStreamingInput.class);
@Override
- public StreamingBatch getBatchWithTimeWindow(String streaming, int id, long startTime, long endTime) {
- try {
+ public StreamingBatch getBatchWithTimeWindow(RealizationType realizationType, String realizationName, int id, long startTime, long endTime) {
+ if (realizationType != RealizationType.CUBE) {
+ throw new IllegalArgumentException("Unsupported realization in KafkaStreamingInput: " + realizationType);
+ }
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ final CubeInstance cube = CubeManager.getInstance(kylinConfig).getCube(realizationName);
+ final String streaming = cube.getFactTable();
+ final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
+ final StreamingConfig streamingConfig = streamingManager.getConfig(streaming);
+ if (streamingConfig == null) {
+ throw new IllegalArgumentException("Table " + streaming + " is not a streaming table.");
+ }
+ if (StreamingConfig.STREAMING_TYPE_KAFKA.equals(streamingConfig.getType())) {
logger.info(String.format("prepare to get streaming batch, name:%s, id:%d, startTime:%d, endTime:%d", streaming, id, startTime, endTime));
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
- final KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(streaming);
- final StreamingParser streamingParser = StreamingParser.getStreamingParser(kafkaConfig);
- final ExecutorService executorService = Executors.newCachedThreadPool();
- final List<Future<List<StreamingMessage>>> futures = Lists.newArrayList();
- for (final KafkaClusterConfig kafkaClusterConfig : kafkaConfig.getKafkaClusterConfigs()) {
- final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
- for (int i = 0; i < partitionCount; ++i) {
- final StreamingMessageProducer producer = new StreamingMessageProducer(kafkaClusterConfig, i, Pair.newPair(startTime, endTime), kafkaConfig.getMargin(), streamingParser);
- final Future<List<StreamingMessage>> future = executorService.submit(producer);
- futures.add(future);
+
+ try {
+ final KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
+ final KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(streaming);
+ final StreamingParser streamingParser = StreamingParser.getStreamingParser(kafkaConfig, realizationType, realizationName);
+ final ExecutorService executorService = Executors.newCachedThreadPool();
+ final List<Future<List<StreamingMessage>>> futures = Lists.newArrayList();
+ for (final KafkaClusterConfig kafkaClusterConfig : kafkaConfig.getKafkaClusterConfigs()) {
+ final int partitionCount = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig).getPartitionIds().size();
+ for (int i = 0; i < partitionCount; ++i) {
+ final StreamingMessageProducer producer = new StreamingMessageProducer(kafkaClusterConfig, i, Pair.newPair(startTime, endTime), kafkaConfig.getMargin(), streamingParser);
+ final Future<List<StreamingMessage>> future = executorService.submit(producer);
+ futures.add(future);
+ }
}
- }
- List<StreamingMessage> messages = Lists.newLinkedList();
- for (Future<List<StreamingMessage>> future : futures) {
- try {
- messages.addAll(future.get());
- } catch (InterruptedException e) {
- logger.warn("this thread should not be interrupted, just ignore", e);
- continue;
- } catch (ExecutionException e) {
- throw new RuntimeException("error when get StreamingMessages",e.getCause());
+ List<StreamingMessage> messages = Lists.newLinkedList();
+ for (Future<List<StreamingMessage>> future : futures) {
+ try {
+ messages.addAll(future.get());
+ } catch (InterruptedException e) {
+ logger.warn("this thread should not be interrupted, just ignore", e);
+ continue;
+ } catch (ExecutionException e) {
+ throw new RuntimeException("error when get StreamingMessages", e.getCause());
+ }
}
+ final Pair<Long, Long> timeRange = Pair.newPair(startTime, endTime);
+ logger.info("finish to get streaming batch, total message count:" + messages.size());
+ return new StreamingBatch(messages, timeRange);
+ } catch (ReflectiveOperationException e) {
+ throw new RuntimeException("failed to create instance of StreamingParser", e);
}
- final Pair<Long, Long> timeRange = Pair.newPair(startTime, endTime);
- logger.info("finish to get streaming batch, total message count:" + messages.size());
- return new StreamingBatch(messages, timeRange);
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("failed to create instance of StreamingParser", e);
+ } else {
+ throw new IllegalArgumentException("kafka is the only supported streaming type.");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9021f17d/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
index 3455f1d..7b326e2 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
@@ -50,6 +50,7 @@ import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.metadata.model.IntermediateColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import com.google.common.base.Function;
@@ -68,9 +69,8 @@ public abstract class StreamingParser {
abstract public boolean filter(StreamingMessage streamingMessage);
- public static StreamingParser getStreamingParser(KafkaConfig kafkaConfig) throws ReflectiveOperationException {
- final String cubeName = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getStreamingConfig(kafkaConfig.getName()).getCubeName();
- final CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
+ public static StreamingParser getStreamingParser(KafkaConfig kafkaConfig, RealizationType realizationType, String realizationName) throws ReflectiveOperationException {
+ final CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(realizationName);
List<TblColRef> columns = Lists.transform(new CubeJoinedFlatTableDesc(cubeInstance.getDescriptor(), null).getColumnList(), new Function<IntermediateColumnDesc, TblColRef>() {
@Nullable
@Override
[36/50] [abbrv] kylin git commit: clean code
Posted by sh...@apache.org.
clean code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e13bba0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e13bba0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e13bba0
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 5e13bba0822cafeb02817e5e59c08a3a4b5020c9
Parents: 4662ada
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 11 14:03:45 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 11 15:00:35 2016 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 26 ++-----
.../java/org/apache/kylin/cube/CubeSegment.java | 31 +++++---
.../org/apache/kylin/cube/cuboid/Cuboid.java | 26 +++----
.../kylin/job/constant/ExecutableConstants.java | 20 -----
.../kylin/job/execution/AbstractExecutable.java | 27 +++----
.../kylin/engine/mr/BatchCubingJobBuilder.java | 25 ++++---
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 48 ++++++------
.../kylin/engine/mr/BatchMergeJobBuilder.java | 13 ++--
.../kylin/engine/mr/BatchMergeJobBuilder2.java | 22 +++---
.../org/apache/kylin/engine/mr/CubingJob.java | 56 ++++----------
.../kylin/engine/mr/JobBuilderSupport.java | 63 +++++++++-------
.../engine/mr/common/AbstractHadoopJob.java | 33 ++++-----
.../kylin/engine/mr/common/BatchConstants.java | 59 ++++++++++-----
.../kylin/engine/mr/common/CubeStatsReader.java | 56 +++++++-------
.../kylin/engine/mr/common/CuboidStatsUtil.java | 2 +-
.../mr/invertedindex/BatchIIJobBuilder.java | 21 +++---
.../mr/invertedindex/InvertedIndexJob.java | 7 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 2 +-
.../engine/mr/steps/CubingExecutableUtil.java | 78 ++++++++++++++++++++
.../kylin/engine/mr/steps/CuboidReducer.java | 2 +-
.../engine/mr/steps/FactDistinctColumnsJob.java | 2 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../mr/steps/FactDistinctColumnsReducer.java | 4 +-
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 2 +-
.../engine/mr/steps/InMemCuboidMapper.java | 2 +-
.../engine/mr/steps/InMemCuboidReducer.java | 2 +-
.../engine/mr/steps/MergeDictionaryStep.java | 47 +-----------
.../engine/mr/steps/MergeStatisticsStep.java | 66 ++---------------
.../kylin/engine/mr/steps/NDCuboidMapper.java | 4 +-
.../engine/mr/steps/SaveStatisticsStep.java | 49 ++----------
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 38 ++--------
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 57 ++------------
.../apache/kylin/engine/spark/SparkCubing.java | 3 +-
.../engine/spark/SparkCubingJobBuilder.java | 6 +-
.../apache/kylin/rest/service/BasicService.java | 40 +---------
.../apache/kylin/rest/service/JobService.java | 51 +++++++------
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../cardinality/HiveColumnCardinalityJob.java | 3 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 46 ++++--------
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 78 ++++++++++----------
.../hbase/steps/HBaseStreamingOutput.java | 2 +-
.../hbase/steps/RangeKeyDistributionJob.java | 10 +--
.../steps/RangeKeyDistributionReducer.java | 20 ++---
43 files changed, 483 insertions(+), 670 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index ccae80b..88ee553 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -73,7 +73,7 @@ abstract public class ResourceStore {
}
return knownImpl;
}
-
+
private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
List<Throwable> es = new ArrayList<Throwable>();
logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
@@ -141,7 +141,7 @@ abstract public class ResourceStore {
RawResource res = getResourceImpl(resPath);
if (res == null)
return null;
-
+
DataInputStream din = new DataInputStream(res.inputStream);
try {
T r = serializer.deserialize(din);
@@ -160,25 +160,9 @@ abstract public class ResourceStore {
final public long getResourceTimestamp(String resPath) throws IOException {
return getResourceTimestampImpl(norm(resPath));
}
-
+
final public <T extends RootPersistentEntity> List<T> getAllResources(String rangeStart, String rangeEnd, Class<T> clazz, Serializer<T> serializer) throws IOException {
- final List<RawResource> allResources = getAllResources(rangeStart, rangeEnd);
- if (allResources.isEmpty()) {
- return Collections.emptyList();
- }
- List<T> result = Lists.newArrayList();
- try {
- for (RawResource rawResource : allResources) {
- final T element = serializer.deserialize(new DataInputStream(rawResource.inputStream));
- element.setLastModified(rawResource.timestamp);
- result.add(element);
- }
- return result;
- } finally {
- for (RawResource rawResource : allResources) {
- IOUtils.closeQuietly(rawResource.inputStream);
- }
- }
+ return getAllResources(rangeStart, rangeEnd, -1L, -1L, clazz, serializer);
}
final public <T extends RootPersistentEntity> List<T> getAllResources(String rangeStart, String rangeEnd, long timeStartInMillis, long timeEndInMillis, Class<T> clazz, Serializer<T> serializer) throws IOException {
@@ -210,7 +194,7 @@ abstract public class ResourceStore {
/** returns 0 if not exists */
abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
-
+
/**
* overwrite a resource without write conflict check
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 67dce73..5b61c10 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -18,12 +18,12 @@
package org.apache.kylin.cube;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonBackReference;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ShardingHash;
@@ -37,11 +37,12 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.IRealizationSegment;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Maps;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, IRealizationSegment {
@@ -76,6 +77,9 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
private Map<Long, Short> cuboidShardNums = Maps.newHashMap();
@JsonProperty("total_shards")
private int totalShards = 0;
+ @JsonProperty("blackout_cuboids")
+ private List<Long> blackoutCuboids = Lists.newArrayList();
+
@JsonProperty("binary_signature")
private String binarySignature; // a hash of cube schema and dictionary ID, used for sanity check
@@ -391,6 +395,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
/**
* get the number of shards where each cuboid will distribute
+ *
* @return
*/
public Short getCuboidShardNum(Long cuboidId) {
@@ -423,6 +428,10 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
return ret;
}
+ public List<Long> getBlackoutCuboids() {
+ return this.blackoutCuboids;
+ }
+
@Override
public IRealization getRealization() {
return cubeInstance;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 16b0287..89f5204 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -18,10 +18,14 @@
package org.apache.kylin.cube.cuboid;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
@@ -31,13 +35,10 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
public class Cuboid implements Comparable<Cuboid> {
@@ -231,8 +232,7 @@ public class Cuboid implements Comparable<Cuboid> {
return true;
}
- hier:
- for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
+ hier: for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
long result = cuboidID & hierarchyMasks.fullMask;
if (result > 0) {
for (long mask : hierarchyMasks.allMasks) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index d370b0d..d3c1003 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -26,21 +26,15 @@ public final class ExecutableConstants {
}
public static final String YARN_APP_ID = "yarn_application_id";
-
public static final String YARN_APP_URL = "yarn_application_tracking_url";
public static final String MR_JOB_ID = "mr_job_id";
public static final String HDFS_BYTES_WRITTEN = "hdfs_bytes_written";
public static final String SOURCE_RECORDS_COUNT = "source_records_count";
public static final String SOURCE_RECORDS_SIZE = "source_records_size";
- public static final String GLOBAL_LISTENER_NAME = "ChainListener";
public static final int DEFAULT_SCHEDULER_INTERVAL_SECONDS = 60;
- public static final String CUBE_JOB_GROUP_NAME = "cube_job_group";
-
- public static final String DAEMON_JOB_GROUP_NAME = "daemon_job_group";
public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary";
-
public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table";
public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid Data";
@@ -57,23 +51,9 @@ public final class ExecutableConstants {
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
-
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
public static final String STEP_NAME_UPDATE_II_INFO = "Update Inverted Index Info";
- public static final String PROP_ENGINE_CONTEXT = "jobengineConfig";
- public static final String PROP_JOB_FLOW = "jobFlow";
- public static final String PROP_JOBINSTANCE_UUID = "jobInstanceUuid";
- public static final String PROP_JOBSTEP_SEQ_ID = "jobStepSequenceID";
- public static final String PROP_COMMAND = "command";
- // public static final String PROP_STORAGE_LOCATION =
- // "storageLocationIdentifier";
- public static final String PROP_JOB_ASYNC = "jobAsync";
- public static final String PROP_JOB_CMD_EXECUTOR = "jobCmdExecutor";
- public static final String PROP_JOB_CMD_OUTPUT = "jobCmdOutput";
- public static final String PROP_JOB_KILLED = "jobKilled";
- public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows";
-
public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 8d5fea5..83c61ae 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -18,10 +18,13 @@
package org.apache.kylin.job.execution;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
@@ -32,12 +35,10 @@ import org.apache.kylin.job.manager.ExecutableManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
*/
@@ -117,12 +118,12 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
}
retry++;
} while (((result != null && result.succeed() == false) || exception != null) && needRetry() == true);
-
+
if (exception != null) {
onExecuteError(exception, executableContext);
throw new ExecuteException(exception);
}
-
+
onExecuteFinished(result, executableContext);
return result;
}
@@ -164,7 +165,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
@Override
public final Map<String, String> getParams() {
- return Collections.unmodifiableMap(this.params);
+ return this.params;
}
public final String getParam(String key) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 3c10c09..45d03d1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -22,6 +22,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
import org.apache.kylin.engine.mr.IMROutput.IMRBatchCubingOutputSide;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.engine.mr.steps.BaseCuboidJob;
import org.apache.kylin.engine.mr.steps.NDCuboidJob;
@@ -92,12 +93,12 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
baseCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_BASE_CUBOID);
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", "FLAT_TABLE"); // marks flat table input
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[0]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
- appendExecCmdParameters(cmd, "level", "0");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0");
baseCuboidStep.setMapReduceParams(cmd.toString());
baseCuboidStep.setMapReduceJobClass(BaseCuboidJob.class);
@@ -113,12 +114,12 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, ((CubeSegment) seg).getCubeDesc().getModel());
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
- appendExecCmdParameters(cmd, "level", "" + (totalRowkeyColumnCount - dimNum));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum));
ndCuboidStep.setMapReduceParams(cmd.toString());
ndCuboidStep.setMapReduceJobClass(NDCuboidJob.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 86439a8..5f4a3ed 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -22,8 +22,10 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.engine.mr.steps.BaseCuboidJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.mr.steps.InMemCuboidJob;
import org.apache.kylin.engine.mr.steps.NDCuboidJob;
import org.apache.kylin.engine.mr.steps.SaveStatisticsStep;
@@ -89,10 +91,10 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
private SaveStatisticsStep createSaveStatisticsStep(String jobId) {
SaveStatisticsStep result = new SaveStatisticsStep();
result.setName(ExecutableConstants.STEP_NAME_SAVE_STATISTICS);
- result.setCubeName(seg.getRealization().getName());
- result.setSegmentId(seg.getUuid());
- result.setStatisticsPath(getStatisticsPath(jobId));
- result.setCubingJobId(jobId);
+ CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+ CubingExecutableUtil.setStatisticsPath(getStatisticsPath(jobId), result.getParams());
+ CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
return result;
}
@@ -105,11 +107,11 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "output", cuboidRootPath);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Cube_Builder_" + seg.getRealization().getName());
- appendExecCmdParameters(cmd, "cubingJobId", jobId);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidRootPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Cube_Builder_" + seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
cubeStep.setMapReduceParams(cmd.toString());
cubeStep.setMapReduceJobClass(InMemCuboidJob.class);
@@ -126,13 +128,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
baseCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_BASE_CUBOID);
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", "FLAT_TABLE"); // marks flat table input
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[0]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
- appendExecCmdParameters(cmd, "level", "0");
- appendExecCmdParameters(cmd, "cubingJobId", jobId);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
baseCuboidStep.setMapReduceParams(cmd.toString());
baseCuboidStep.setMapReduceJobClass(BaseCuboidJob.class);
@@ -148,13 +150,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, ((CubeSegment) seg).getCubeDesc().getModel());
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
- appendExecCmdParameters(cmd, "level", "" + (totalRowkeyColumnCount - dimNum));
- appendExecCmdParameters(cmd, "cubingJobId", jobId);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
ndCuboidStep.setMapReduceParams(cmd.toString());
ndCuboidStep.setMapReduceJobClass(NDCuboidJob.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
index 831aa9d..6f1d445 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.IMROutput.IMRBatchMergeOutputSide;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.engine.mr.steps.MergeCuboidJob;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -48,7 +49,7 @@ public class BatchMergeJobBuilder extends JobBuilderSupport {
public CubingJob build() {
logger.info("MR_V1 new job to MERGE segment " + seg);
- final CubeSegment cubeSegment = (CubeSegment)seg;
+ final CubeSegment cubeSegment = (CubeSegment) seg;
final CubingJob result = CubingJob.createMergeJob(cubeSegment, submitter, config);
final String jobId = result.getId();
@@ -84,11 +85,11 @@ public class BatchMergeJobBuilder extends JobBuilderSupport {
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg.getRealization().getDataModelDesc());
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", outputPath);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getCubeInstance().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
mergeCuboidDataStep.setMapReduceParams(cmd.toString());
mergeCuboidDataStep.setMapReduceJobClass(MergeCuboidJob.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index 008d489..e151674 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -22,7 +22,9 @@ import java.util.List;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.mr.steps.MergeCuboidJob;
import org.apache.kylin.engine.mr.steps.MergeStatisticsStep;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -79,10 +81,12 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
private MergeStatisticsStep createMergeStatisticsStep(CubeSegment seg, List<String> mergingSegmentIds, String mergedStatisticsFolder) {
MergeStatisticsStep result = new MergeStatisticsStep();
result.setName(ExecutableConstants.STEP_NAME_MERGE_STATISTICS);
- result.setCubeName(seg.getCubeInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setMergingSegmentIds(mergingSegmentIds);
- result.setMergedStatisticsPath(mergedStatisticsFolder);
+
+ CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+ CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams());
+ CubingExecutableUtil.setMergedStatisticsPath(mergedStatisticsFolder, result.getParams());
+
return result;
}
@@ -92,11 +96,11 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg.getRealization().getDataModelDesc());
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", outputPath);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getCubeInstance().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
mergeCuboidDataStep.setMapReduceParams(cmd.toString());
mergeCuboidDataStep.setMapReduceJobClass(MergeCuboidJob.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 979ff75..0325a09 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -24,17 +24,16 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
-import java.util.List;
import java.util.TimeZone;
import java.util.regex.Matcher;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -47,8 +46,8 @@ import org.apache.kylin.job.execution.Output;
/**
*/
public class CubingJob extends DefaultChainedExecutable {
-
- public static enum AlgorithmEnum {
+
+ public enum AlgorithmEnum {
LAYER, INMEM
}
@@ -58,9 +57,6 @@ public class CubingJob extends DefaultChainedExecutable {
public static final String CUBE_SIZE_BYTES = "byteSizeBytes";
public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
- private static final String CUBE_INSTANCE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
-
public static CubingJob createBuildJob(CubeSegment seg, String submitter, JobEngineConfig config) {
return initCubingJob(seg, "BUILD", submitter, config);
}
@@ -73,8 +69,8 @@ public class CubingJob extends DefaultChainedExecutable {
CubingJob result = new CubingJob();
SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
- result.setCubeName(seg.getCubeInstance().getName());
- result.setSegmentId(seg.getUuid());
+ CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis())));
result.setSubmitter(submitter);
result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
@@ -85,29 +81,9 @@ public class CubingJob extends DefaultChainedExecutable {
super();
}
- void setCubeName(String name) {
- setParam(CUBE_INSTANCE_NAME, name);
- }
-
- public String getCubeName() {
- return getParam(CUBE_INSTANCE_NAME);
- }
-
- void setSegmentIds(List<String> segmentIds) {
- setParam(SEGMENT_ID, StringUtils.join(segmentIds, ","));
- }
-
- void setSegmentId(String segmentId) {
- setParam(SEGMENT_ID, segmentId);
- }
-
- public String getSegmentIds() {
- return getParam(SEGMENT_ID);
- }
-
@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
- CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig()).getCube(getCubeName());
+ CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig()).getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final Output output = jobService.getOutput(getId());
String logMsg;
state = output.getState();
@@ -131,7 +107,7 @@ public class CubingJob extends DefaultChainedExecutable {
String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
content = content.replaceAll("\\$\\{job_name\\}", getName());
content = content.replaceAll("\\$\\{result\\}", state.toString());
- content = content.replaceAll("\\$\\{cube_name\\}", getCubeName());
+ content = content.replaceAll("\\$\\{cube_name\\}", CubingExecutableUtil.getCubeName(this.getParams()));
content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount()));
content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
content = content.replaceAll("\\$\\{duration\\}", getDuration() / 60000 + "mins");
@@ -147,7 +123,7 @@ public class CubingJob extends DefaultChainedExecutable {
logger.warn(e.getLocalizedMessage(), e);
}
- String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName();
+ String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + CubingExecutableUtil.getCubeName(this.getParams());
return Pair.of(title, content);
}
@@ -174,11 +150,11 @@ public class CubingJob extends DefaultChainedExecutable {
public void setMapReduceWaitTime(long t) {
addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
}
-
+
public void setAlgorithm(AlgorithmEnum alg) {
addExtraInfo("algorithm", alg.name());
}
-
+
public AlgorithmEnum getAlgorithm() {
String alg = getExtraInfo().get("algorithm");
return alg == null ? null : AlgorithmEnum.valueOf(alg);
@@ -187,11 +163,11 @@ public class CubingJob extends DefaultChainedExecutable {
public boolean isLayerCubing() {
return AlgorithmEnum.LAYER == getAlgorithm();
}
-
+
public boolean isInMemCubing() {
return AlgorithmEnum.INMEM == getAlgorithm();
}
-
+
public long findSourceRecordCount() {
return Long.parseLong(findExtraInfo(SOURCE_RECORD_COUNT, "0"));
}
@@ -204,7 +180,7 @@ public class CubingJob extends DefaultChainedExecutable {
// look for the info BACKWARD, let the last step that claims the cube size win
return Long.parseLong(findExtraInfoBackward(CUBE_SIZE_BYTES, "0"));
}
-
+
public String findExtraInfo(String key, String dft) {
return findExtraInfo(key, dft, false);
}
@@ -212,14 +188,14 @@ public class CubingJob extends DefaultChainedExecutable {
public String findExtraInfoBackward(String key, String dft) {
return findExtraInfo(key, dft, true);
}
-
+
private String findExtraInfo(String key, String dft, boolean backward) {
ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks());
-
+
if (backward) {
Collections.reverse(tasks);
}
-
+
for (AbstractExecutable child : tasks) {
Output output = executableManager.getOutput(child.getId());
String value = output.getExtra().get(key);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index e3b07d8..a3fef8e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -23,21 +23,23 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.engine.mr.invertedindex.UpdateIIInfoAfterBuildStep;
import org.apache.kylin.engine.mr.steps.CreateDictionaryJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob;
import org.apache.kylin.engine.mr.steps.MergeDictionaryStep;
import org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep;
import org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterMergeStep;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
-
-import com.google.common.base.Preconditions;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.realization.IRealizationSegment;
+import com.google.common.base.Preconditions;
+
/**
* Hold reusable steps for builders.
*/
@@ -67,14 +69,14 @@ public class JobBuilderSupport {
result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS);
result.setMapReduceJobClass(FactDistinctColumnsJob.class);
StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, ((CubeSegment)seg).getCubeDesc().getModel());
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "output", getFactDistinctColumnsPath(jobId));
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "statisticsenabled", String.valueOf(withStats));
- appendExecCmdParameters(cmd, "statisticsoutput", getStatisticsPath(jobId));
- appendExecCmdParameters(cmd, "statisticssamplingpercent", String.valueOf(config.getConfig().getCubingInMemSamplingPercent()));
- appendExecCmdParameters(cmd, "jobname", "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step");
+ appendMapReduceParameters(cmd, ((CubeSegment) seg).getCubeDesc().getModel());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getFactDistinctColumnsPath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_ENABLED, String.valueOf(withStats));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_OUTPUT, getStatisticsPath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent()));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step");
result.setMapReduceParams(cmd.toString());
return result;
@@ -85,9 +87,9 @@ public class JobBuilderSupport {
HadoopShellExecutable buildDictionaryStep = new HadoopShellExecutable();
buildDictionaryStep.setName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY);
StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", getFactDistinctColumnsPath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getFactDistinctColumnsPath(jobId));
buildDictionaryStep.setJobParams(cmd.toString());
buildDictionaryStep.setJobClass(CreateDictionaryJob.class);
@@ -95,34 +97,38 @@ public class JobBuilderSupport {
}
public UpdateCubeInfoAfterBuildStep createUpdateCubeInfoAfterBuildStep(String jobId) {
- final UpdateCubeInfoAfterBuildStep updateCubeInfoStep = new UpdateCubeInfoAfterBuildStep();
- updateCubeInfoStep.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
- updateCubeInfoStep.setCubeName(seg.getRealization().getName());
- updateCubeInfoStep.setSegmentId(seg.getUuid());
- updateCubeInfoStep.setCubingJobId(jobId);
- return updateCubeInfoStep;
+ final UpdateCubeInfoAfterBuildStep result = new UpdateCubeInfoAfterBuildStep();
+ result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
+
+ CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+ CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
+
+ return result;
}
public MergeDictionaryStep createMergeDictionaryStep(List<String> mergingSegmentIds) {
MergeDictionaryStep result = new MergeDictionaryStep();
result.setName(ExecutableConstants.STEP_NAME_MERGE_DICTIONARY);
- result.setCubeName(seg.getRealization().getName());
- result.setSegmentId(seg.getUuid());
- result.setMergingSegmentIds(mergingSegmentIds);
+
+ CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+ CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams());
+
return result;
}
public UpdateCubeInfoAfterMergeStep createUpdateCubeInfoAfterMergeStep(List<String> mergingSegmentIds, String jobId) {
UpdateCubeInfoAfterMergeStep result = new UpdateCubeInfoAfterMergeStep();
result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
- result.setCubeName(seg.getRealization().getName());
- result.setSegmentId(seg.getUuid());
- result.setMergingSegmentIds(mergingSegmentIds);
- result.setCubingJobId(jobId);
- return result;
- }
+ CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
+ CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
+ CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
+ CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams());
+ return result;
+ }
public UpdateIIInfoAfterBuildStep createUpdateIIInfoAfterBuildStep(String jobId) {
final UpdateIIInfoAfterBuildStep updateIIInfoStep = new UpdateIIInfoAfterBuildStep();
@@ -141,6 +147,7 @@ public class JobBuilderSupport {
public String getRealizationRootPath(String jobId) {
return getJobWorkingDir(jobId) + "/" + seg.getRealization().getName();
}
+
public String getCuboidRootPath(String jobId) {
return getRealizationRootPath(jobId) + "/cuboid/";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 61983d5..fe60ca8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -23,7 +23,7 @@ package org.apache.kylin.engine.mr.common;
*
*/
-import static org.apache.hadoop.util.StringUtils.*;
+import static org.apache.hadoop.util.StringUtils.formatTime;
import java.io.File;
import java.io.IOException;
@@ -74,22 +74,21 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
- protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Job name. For exmaple, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create("jobname");
- protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create("cubename");
- protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName("cubingJobId").hasArg().isRequired(false).withDescription("ID of cubing job executable").create("cubingJobId");
- protected static final Option OPTION_II_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create("iiname");
- protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Cube segment name)").create("segmentname");
- protected static final Option OPTION_TABLE_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Hive table name.").create("tablename");
- protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Input path").create("input");
- protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName("inputformat").hasArg().isRequired(false).withDescription("Input format").create("inputformat");
- protected static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Output path").create("output");
- protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName("level").hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create("level");
- protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Partition file path.").create("partitions");
- protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName("htable name").hasArg().isRequired(true).withDescription("HTable name").create("htablename");
-
- protected static final Option OPTION_STATISTICS_ENABLED = OptionBuilder.withArgName("statisticsenabled").hasArg().isRequired(false).withDescription("Statistics enabled").create("statisticsenabled");
- protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName("statisticsoutput").hasArg().isRequired(false).withDescription("Statistics output").create("statisticsoutput");
- protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName("statisticssamplingpercent").hasArg().isRequired(false).withDescription("Statistics sampling percentage").create("statisticssamplingpercent");
+ protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
+ protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
+ protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
+ protected static final Option OPTION_II_NAME = OptionBuilder.withArgName(BatchConstants.ARG_II_NAME).hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create(BatchConstants.ARG_II_NAME);
+ protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName( BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create( BatchConstants.ARG_SEGMENT_NAME);
+ protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT);
+ protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName(BatchConstants.ARG_INPUT_FORMAT).hasArg().isRequired(false).withDescription("Input format").create(BatchConstants.ARG_INPUT_FORMAT);
+ protected static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Output path").create(BatchConstants.ARG_OUTPUT);
+ protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName(BatchConstants.ARG_LEVEL).hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create(BatchConstants.ARG_LEVEL);
+ protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName(BatchConstants.ARG_PARTITION).hasArg().isRequired(true).withDescription("Partition file path.").create(BatchConstants.ARG_PARTITION);
+ protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_HTABLE_NAME).hasArg().isRequired(true).withDescription("HTable name").create(BatchConstants.ARG_HTABLE_NAME);
+
+ protected static final Option OPTION_STATISTICS_ENABLED = OptionBuilder.withArgName(BatchConstants.ARG_STATS_ENABLED).hasArg().isRequired(false).withDescription("Statistics enabled").create(BatchConstants.ARG_STATS_ENABLED);
+ protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_OUTPUT).hasArg().isRequired(false).withDescription("Statistics output").create(BatchConstants.ARG_STATS_OUTPUT);
+ protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_SAMPLING_PERCENT).hasArg().isRequired(false).withDescription("Statistics sampling percentage").create(BatchConstants.ARG_STATS_SAMPLING_PERCENT);
protected String name;
protected boolean isAsync = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 6943f18..a614f4b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -20,8 +20,15 @@ package org.apache.kylin.engine.mr.common;
public interface BatchConstants {
+ /**
+ * source data config
+ */
char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
+ /**
+ * ConFiGuration entry names for MR jobs
+ */
+
String CFG_CUBE_NAME = "cube.name";
String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
String CFG_CUBE_CUBOID_LEVEL = "cube.cuboid.level";
@@ -29,31 +36,47 @@ public interface BatchConstants {
String CFG_II_NAME = "ii.name";
String CFG_II_SEGMENT_NAME = "ii.segment.name";
- String OUTPUT_PATH = "output.path";
-
- String TABLE_NAME = "table.name";
- String TABLE_COLUMNS = "table.columns";
-
+ String CFG_OUTPUT_PATH = "output.path";
+ String CFG_TABLE_NAME = "table.name";
String CFG_IS_MERGE = "is.merge";
String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
+ String CFG_REGION_NUMBER_MIN = "region.number.min";
+ String CFG_REGION_NUMBER_MAX = "region.number.max";
+ String CFG_REGION_SPLIT_SIZE = "region.split.size";
+ String CFG_HFILE_SIZE_GB = "hfile.size.gb";
- String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
-
- String REGION_NUMBER_MIN = "region.number.min";
- String REGION_NUMBER_MAX = "region.number.max";
- String REGION_SPLIT_SIZE = "region.split.size";
- String HFILE_SIZE_GB = "hfile.size.gb";
-
String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/";
String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/";
-
+
String CFG_STATISTICS_LOCAL_DIR = CFG_KYLIN_LOCAL_TEMP_DIR + "cuboidstatistics/";
String CFG_STATISTICS_ENABLED = "statistics.enabled";
- String CFG_STATISTICS_OUTPUT = "statistics.ouput";
+ String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for compatibility issue better not change it
String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
- String CFG_STATISTICS_CUBE_ESTIMATION = "cube_statistics.txt";
- String CFG_STATISTICS_CUBOID_ESTIMATION = "cuboid_statistics.seq";
+ String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
+ String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
- int COUNTER_MAX = 100000;
- int ERROR_RECORD_THRESHOLD = 100;
+ /**
+ * command line ARGuments
+ */
+ String ARG_INPUT = "input";
+ String ARG_OUTPUT = "output";
+ String ARG_JOB_NAME = "jobname";
+ String ARG_CUBING_JOB_ID = "cubingJobId";
+ String ARG_CUBE_NAME = "cubename";
+ String ARG_II_NAME = "iiname";
+ String ARG_SEGMENT_NAME = "segmentname";
+ String ARG_PARTITION = "partitions";
+ String ARG_STATS_ENABLED= "statisticsenabled";
+ String ARG_STATS_OUTPUT= "statisticsoutput";
+ String ARG_STATS_SAMPLING_PERCENT= "statisticssamplingpercent";
+ String ARG_HTABLE_NAME= "htablename";
+ String ARG_INPUT_FORMAT= "inputformat";
+ String ARG_LEVEL= "level";
+
+ /**
+ * logger and counter
+ */
+ String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
+ int NORMAL_RECORD_LOG_THRESHOLD = 100000;
+ int ERROR_RECORD_LOG_THRESHOLD = 100;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 57e93c3..00b9e32 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -18,19 +18,8 @@
package org.apache.kylin.engine.mr.common;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
@@ -61,9 +50,16 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
/**
* This should be in cube module. It's here in engine-mr because currently stats
@@ -76,7 +72,7 @@ public class CubeStatsReader {
final CubeSegment seg;
final int samplingPercentage;
final double mapperOverlapRatioOfFirstBuild; // only makes sense for the first build, is meaningless after merge
- final Map<Long, HyperLogLogPlusCounter> cuboidRowCountMap;
+ final Map<Long, HyperLogLogPlusCounter> cuboidRowEstimatesHLL;
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
ResourceStore store = ResourceStore.getStore(kylinConfig);
@@ -112,7 +108,7 @@ public class CubeStatsReader {
this.seg = cubeSegment;
this.samplingPercentage = percentage;
this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
- this.cuboidRowCountMap = counterMap;
+ this.cuboidRowEstimatesHLL = counterMap;
} finally {
IOUtils.closeStream(reader);
@@ -133,13 +129,13 @@ public class CubeStatsReader {
return tempFile;
}
- public Map<Long, Long> getCuboidRowCountMap() {
- return getCuboidRowCountMapFromSampling(cuboidRowCountMap, samplingPercentage);
+ public Map<Long, Long> getCuboidRowEstimatesHLL() {
+ return getCuboidRowCountMapFromSampling(cuboidRowEstimatesHLL, samplingPercentage);
}
// return map of Cuboid ID => MB
public Map<Long, Double> getCuboidSizeMap() {
- return getCuboidSizeMapFromRowCount(seg, getCuboidRowCountMap());
+ return getCuboidSizeMapFromRowCount(seg, getCuboidRowEstimatesHLL());
}
public double getMapperOverlapRatioOfFirstBuild() {
@@ -147,15 +143,13 @@ public class CubeStatsReader {
}
public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HyperLogLogPlusCounter> hllcMap, int samplingPercentage) {
- return Maps.transformValues(hllcMap, new Function<HyperLogLogPlusCounter, Long>() {
- @Nullable
- @Override
- public Long apply(HyperLogLogPlusCounter input) {
- // No need to adjust according sampling percentage. Assumption is that data set is far
- // more than cardinality. Even a percentage of the data should already see all cardinalities.
- return input.getCountEstimate();
- }
- });
+ Map<Long, Long> cuboidRowCountMap = Maps.newHashMap();
+ for (Map.Entry<Long, HyperLogLogPlusCounter> entry : hllcMap.entrySet()) {
+ // No need to adjust according sampling percentage. Assumption is that data set is far
+ // more than cardinality. Even a percentage of the data should already see all cardinalities.
+ cuboidRowCountMap.put(entry.getKey(), entry.getValue().getCountEstimate());
+ }
+ return cuboidRowCountMap;
}
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap) {
@@ -219,7 +213,7 @@ public class CubeStatsReader {
}
private void print(PrintWriter out) {
- Map<Long, Long> cuboidRows = getCuboidRowCountMap();
+ Map<Long, Long> cuboidRows = getCuboidRowEstimatesHLL();
Map<Long, Double> cuboidSizes = getCuboidSizeMap();
List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet());
Collections.sort(cuboids);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
index 02fe0f0..cb4b1cb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
@@ -43,7 +43,7 @@ public class CuboidStatsUtil {
}
public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage, double mapperOverlapRatio) throws IOException {
- Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
List<Long> allCuboids = new ArrayList<Long>();
allCuboids.addAll(cuboidHLLMap.keySet());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
index e7501b8..4841e64 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
@@ -22,6 +22,7 @@ import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
import org.apache.kylin.engine.mr.IMROutput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.MRUtil;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -29,9 +30,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BatchIIJobBuilder extends JobBuilderSupport {
-
+
private static final Logger logger = LoggerFactory.getLogger(BatchIIJobBuilder.class);
-
+
private final IMRBatchCubingInputSide inputSide;
private final IMROutput.IMRBatchInvertedIndexingOutputSide outputSide;
@@ -44,15 +45,15 @@ public class BatchIIJobBuilder extends JobBuilderSupport {
public IIJob build() {
logger.info("MR new job to BUILD segment " + seg);
- final IIJob result = IIJob.createBuildJob((IISegment)seg, submitter, config);
+ final IIJob result = IIJob.createBuildJob((IISegment) seg, submitter, config);
final String jobId = result.getId();
-
+
final String iiRootPath = getRealizationRootPath(jobId) + "/";
// Phase 1: Create Flat Table
inputSide.addStepPhase1_CreateFlatTable(result);
-
+
// Phase 2: Build Inverted Index
- result.addTask(createInvertedIndexStep((IISegment)seg, iiRootPath));
+ result.addTask(createInvertedIndexStep((IISegment) seg, iiRootPath));
outputSide.addStepPhase3_BuildII(result, iiRootPath);
// Phase 3: Update Metadata & Cleanup
@@ -71,13 +72,13 @@ public class BatchIIJobBuilder extends JobBuilderSupport {
buildIIStep.setName(ExecutableConstants.STEP_NAME_BUILD_II);
- appendExecCmdParameters(cmd, "iiname", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "output", iiOutputTempPath);
- appendExecCmdParameters(cmd, "jobname", ExecutableConstants.STEP_NAME_BUILD_II);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, iiOutputTempPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, ExecutableConstants.STEP_NAME_BUILD_II);
buildIIStep.setMapReduceParams(cmd.toString());
buildIIStep.setMapReduceJobClass(InvertedIndexJob.class);
return buildIIStep;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
index 27505e6..9ea2411 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
@@ -19,14 +19,12 @@
package org.apache.kylin.engine.mr.invertedindex;
import java.io.IOException;
-import java.util.ArrayList;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
@@ -39,8 +37,6 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,7 +102,6 @@ public class InvertedIndexJob extends AbstractHadoopJob {
conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName());
}
-
private void setupMapper(IISegment segment) throws IOException {
IMRInput.IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(segment).getFlatTableInputFormat();
@@ -127,7 +122,7 @@ public class InvertedIndexJob extends AbstractHadoopJob {
FileOutputFormat.setOutputPath(job, output);
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
+ job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
deletePath(job.getConfiguration(), output);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 3dddece..5fb9098 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -195,7 +195,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
// TODO expose errorRecordCounter as hadoop counter
errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
+ if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
if (ex instanceof IOException)
throw (IOException) ex;
else if (ex instanceof RuntimeException)
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
new file mode 100644
index 0000000..71f27f2
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
@@ -0,0 +1,78 @@
+package org.apache.kylin.engine.mr.steps;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.Lists;
+
+public class CubingExecutableUtil {
+
+ public static final String CUBE_NAME = "cubeName";
+ public static final String SEGMENT_ID = "segmentId";
+ public static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
+ public static final String STATISTICS_PATH = "statisticsPath";
+ public static final String CUBING_JOB_ID = "cubingJobId";
+ public static final String MERGED_STATISTICS_PATH = "mergedStatisticsPath";
+
+ public static void setStatisticsPath(String path, Map<String, String> params) {
+ params.put(STATISTICS_PATH, path);
+ }
+
+ public static String getStatisticsPath(Map<String, String> params) {
+ return params.get(STATISTICS_PATH);
+ }
+
+ public static void setCubeName(String cubeName, Map<String, String> params) {
+ params.put(CUBE_NAME, cubeName);
+ }
+
+ public static String getCubeName(Map<String, String> params) {
+ return params.get(CUBE_NAME);
+ }
+
+ public static void setSegmentId(String segmentId, Map<String, String> params) {
+ params.put(SEGMENT_ID, segmentId);
+ }
+
+ public static String getSegmentId(Map<String, String> params) {
+ return params.get(SEGMENT_ID);
+ }
+
+ public static void setMergingSegmentIds(List<String> ids, Map<String, String> params) {
+ params.put(MERGING_SEGMENT_IDS, StringUtils.join(ids, ","));
+ }
+
+ public static List<String> getMergingSegmentIds(Map<String, String> params) {
+ final String ids = params.get(MERGING_SEGMENT_IDS);
+ if (ids != null) {
+ final String[] splitted = StringUtils.split(ids, ",");
+ ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
+ for (String id : splitted) {
+ result.add(id);
+ }
+ return result;
+ } else {
+ return Collections.emptyList();
+ }
+ }
+
+ public static void setCubingJobId(String id, Map<String, String> params) {
+ params.put(CUBING_JOB_ID, id);
+ }
+
+ public static String getCubingJobId(Map<String, String> params) {
+ return params.get(CUBING_JOB_ID);
+ }
+
+ public static void setMergedStatisticsPath(String path, Map<String, String> params) {
+ params.put(MERGED_STATISTICS_PATH, path);
+ }
+
+ public static String getMergedStatisticsPath(Map<String, String> params) {
+ return params.get(MERGED_STATISTICS_PATH);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index 4dbb53e..f263d99 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -108,7 +108,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
context.write(key, outputValue);
counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
+ if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Handled " + counter + " records!");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 1373e2c..2dabb7a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -125,7 +125,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.setNumReduceTasks(numberOfReducers);
FileOutputFormat.setOutputPath(job, output);
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
+ job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
deletePath(job.getConfiguration(), output);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 1412dfb..cc7b6df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -71,7 +71,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
ex.printStackTrace(System.err);
errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
+ if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) {
if (ex instanceof IOException)
throw (IOException) ex;
else if (ex instanceof RuntimeException)
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index f3e0290..f43834d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -128,7 +128,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
- final String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
+ final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
final Path outputFile = new Path(outputPath, col.getName());
FSDataOutputStream out = null;
@@ -176,7 +176,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
private void writeMapperAndCuboidStatistics(Context context) throws IOException {
Configuration conf = context.getConfiguration();
FileSystem fs = FileSystem.get(conf);
- FSDataOutputStream out = fs.create(new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION));
+ FSDataOutputStream out = fs.create(new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME));
try {
String msg;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
index e2a49df..8f5557d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
@@ -41,7 +41,7 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN, O
@Override
public void map(KEYIN key, Object value, Context context) throws IOException, InterruptedException {
counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
+ if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Handled " + counter + " records!");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index b094b98..e42a6b5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -100,7 +100,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
while (!future.isDone()) {
if (queue.offer(rowAsList, 1, TimeUnit.SECONDS)) {
counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
+ if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Handled " + counter + " records!");
}
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index 9beacbb..e72c38b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -85,7 +85,7 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra
context.write(outputKey, outputValue);
counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
+ if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Handled " + counter + " records!");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index f247a8f..264ba9b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -25,7 +25,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -44,10 +43,6 @@ import com.google.common.collect.Lists;
public class MergeDictionaryStep extends AbstractExecutable {
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
-
public MergeDictionaryStep() {
super();
}
@@ -56,8 +51,8 @@ public class MergeDictionaryStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig conf = context.getConfig();
final CubeManager mgr = CubeManager.getInstance(conf);
- final CubeInstance cube = mgr.getCube(getCubeName());
- final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
+ final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
Collections.sort(mergingSegments);
@@ -79,7 +74,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
}
private List<CubeSegment> getMergingSegments(CubeInstance cube) {
- List<String> mergingSegmentIds = getMergingSegmentIds();
+ List<String> mergingSegmentIds = CubingExecutableUtil.getMergingSegmentIds(this.getParams());
List<CubeSegment> result = Lists.newArrayListWithCapacity(mergingSegmentIds.size());
for (String id : mergingSegmentIds) {
result.add(cube.getSegmentById(id));
@@ -111,7 +106,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
CubeDesc cubeDesc = cube.getDescriptor();
for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
- String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(),true, col).getTable();
+ String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
colsNeedMeringDict.add(col);
} else {
@@ -165,38 +160,4 @@ public class MergeDictionaryStep extends AbstractExecutable {
}
}
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setMergingSegmentIds(List<String> ids) {
- setParam(MERGING_SEGMENT_IDS, StringUtils.join(ids, ","));
- }
-
- private List<String> getMergingSegmentIds() {
- final String ids = getParam(MERGING_SEGMENT_IDS);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
}
[46/50] [abbrv] kylin git commit: KYLIN-1226 use low memory config
for sandbox CI
Posted by sh...@apache.org.
KYLIN-1226 use low memory config for sandbox CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4e5a936c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4e5a936c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4e5a936c
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 4e5a936c22997c3ba081a229053b93ea27c7ab46
Parents: 4aeb76b
Author: Yang Li <li...@apache.org>
Authored: Sat Mar 12 11:01:12 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Mar 12 11:01:12 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
.../java/org/apache/kylin/common/KylinConfigBase.java | 12 +++++++++---
.../org/apache/kylin/rest/util/Log4jConfigListener.java | 4 +---
3 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4e5a936c/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index e87494d..68678cc 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -161,7 +161,7 @@ kylin.web.contact_mail=
###########################config info for front#######################
#env DEV|QA|PROD
-deploy.env=DEV
+deploy.env=QA
###########################deprecated configs#######################
kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/4e5a936c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6a47321..e3a73e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -119,6 +119,10 @@ public class KylinConfigBase implements Serializable {
// ============================================================================
+ public boolean isDevEnv() {
+ return "DEV".equals(getOptional("deploy.env", "DEV"));
+ }
+
public String getMetadataUrl() {
return getOptional("kylin.metadata.url");
}
@@ -485,10 +489,12 @@ public class KylinConfigBase implements Serializable {
}
public Map<String, String> getCubingInMemMRJobConfOverride() {
- String str = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", //
- "mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072; mapreduce.task.io.sort.mb=200");
+ // in-mem cubing requires big memory, however dev env (sandbox) may not have that much
+ String defaultOverride = isDevEnv() ? "" : "mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072; mapreduce.task.io.sort.mb=200";
+ String override = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", defaultOverride);
+
Map<String, String> result = Maps.newHashMap();
- for (String pair : str.split(";")) {
+ for (String pair : override.split(";")) {
int cut = pair.indexOf('=');
if (cut < 0)
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4e5a936c/server/src/main/java/org/apache/kylin/rest/util/Log4jConfigListener.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/Log4jConfigListener.java b/server/src/main/java/org/apache/kylin/rest/util/Log4jConfigListener.java
index 23b327a..7e79511 100644
--- a/server/src/main/java/org/apache/kylin/rest/util/Log4jConfigListener.java
+++ b/server/src/main/java/org/apache/kylin/rest/util/Log4jConfigListener.java
@@ -27,9 +27,7 @@ public class Log4jConfigListener extends org.springframework.web.util.Log4jConfi
private boolean isDebugTomcat;
public Log4jConfigListener() {
- // check if is DebugTomcat
- String property = System.getProperty(KylinConfig.KYLIN_CONF);
- this.isDebugTomcat = property != null && property.contains("examples/test_case_data/sandbox");
+ this.isDebugTomcat = KylinConfig.getInstanceFromEnv().isDevEnv();
}
@Override
[14/50] [abbrv] kylin git commit: KYLIN-1387 should support empty
segment
Posted by sh...@apache.org.
KYLIN-1387 should support empty segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9352e5a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9352e5a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9352e5a9
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 9352e5a9e87e8b7bae41303ac1556ca7b7dc0023
Parents: 929c7a4
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 09:42:34 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:19:54 2016 +0800
----------------------------------------------------------------------
.../hbase/steps/SequenceFileCuboidWriter.java | 48 +++++++++++---------
1 file changed, 26 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9352e5a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 4d76522..8c2d5e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -25,35 +25,39 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
public SequenceFileCuboidWriter(CubeDesc cubeDesc, CubeSegment segment) {
super(cubeDesc, segment);
+ try {
+ initiate();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
-
- @Override
- protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
+ protected void initiate() throws IOException {
if (writer == null) {
- synchronized (SequenceFileCuboidWriter.class) {
- if (writer == null) {
- JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
- String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
- Path cuboidPath = new Path(cuboidRoot);
- FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
- try {
- if (fs.exists(cuboidPath)) {
- fs.delete(cuboidPath, true);
- }
-
- fs.mkdirs(cuboidPath);
- } finally {
- IOUtils.closeQuietly(fs);
- }
-
- Path cuboidFile = new Path(cuboidPath, "data.seq");
- logger.debug("Cuboid is written to " + cuboidFile);
- writer = SequenceFile.createWriter(HadoopUtil.getCurrentConfiguration(), SequenceFile.Writer.file(cuboidFile), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class));
+ JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
+ String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
+ Path cuboidPath = new Path(cuboidRoot);
+ FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
+ try {
+ if (fs.exists(cuboidPath)) {
+ fs.delete(cuboidPath, true);
}
+
+ fs.mkdirs(cuboidPath);
+ } finally {
+ IOUtils.closeQuietly(fs);
}
+
+ Path cuboidFile = new Path(cuboidPath, "data.seq");
+ logger.debug("Cuboid is written to " + cuboidFile);
+ writer = SequenceFile.createWriter(HadoopUtil.getCurrentConfiguration(), SequenceFile.Writer.file(cuboidFile), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class));
}
+ }
+
+ @Override
+ protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
+
Text outputValue = new Text();
Text outputKey = new Text();
outputKey.set(key.array(), key.offset(), key.length());
[30/50] [abbrv] kylin git commit: KYLIN-1431 load streaming table ui
update add cluster info when add table #2
Posted by sh...@apache.org.
KYLIN-1431 load streaming table ui update add cluster info when add table #2
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/376d695c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/376d695c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/376d695c
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 376d695c5893c482eaba6853929b8a34276ba964
Parents: 890577d
Author: Jason <ji...@163.com>
Authored: Thu Mar 10 20:50:25 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Mar 11 10:58:35 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/TableDesc.java | 4 +
.../rest/controller/StreamingController.java | 105 ++++++++++---
.../kylin/rest/request/StreamingRequest.java | 3 +-
.../kylin/rest/response/TableDescResponse.java | 1 +
webapp/app/js/controllers/cubeEdit.js | 101 ++----------
webapp/app/js/controllers/modelSchema.js | 2 +-
webapp/app/js/controllers/sourceMeta.js | 154 +++++++++++++++----
webapp/app/js/controllers/streamingConfig.js | 31 +++-
webapp/app/js/model/streamingModel.js | 3 +-
webapp/app/js/model/tableModel.js | 7 +-
webapp/app/partials/cubeDesigner/info.html | 10 +-
.../partials/cubeDesigner/kafkaBasicConfig.html | 4 +-
.../partials/cubeDesigner/streamingConfig.html | 69 ++++++---
.../app/partials/modelDesigner/data_model.html | 8 +-
.../app/partials/modelDesigner/model_info.html | 6 +-
.../app/partials/tables/loadStreamingTable.html | 19 +--
webapp/app/partials/tables/table_detail.html | 12 +-
17 files changed, 343 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index d5e4dbb..65d85dd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -180,4 +180,8 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public int getSourceType() {
return sourceType;
}
+
+ public void setSourceType(int sourceType) {
+ this.sourceType = sourceType;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index ecd7571..932ad51 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -22,13 +22,17 @@ import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.StreamingRequest;
+import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.KafkaConfigService;
import org.apache.kylin.rest.service.StreamingService;
import org.apache.kylin.source.kafka.config.KafkaConfig;
@@ -40,7 +44,9 @@ import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
/**
@@ -57,6 +63,9 @@ public class StreamingController extends BasicController {
private StreamingService streamingService;
@Autowired
private KafkaConfigService kafkaConfigService;
+ @Autowired
+ private CubeService cubeMgmtService;
+
@RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
@ResponseBody
@@ -89,36 +98,69 @@ public class StreamingController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
public StreamingRequest saveStreamingConfig(@RequestBody StreamingRequest streamingRequest) {
- //Update Model
+
+ String project = streamingRequest.getProject();
+ TableDesc tableDesc = deserializeTableDesc(streamingRequest);
StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
- if (streamingConfig == null ||kafkaConfig == null) {
- return streamingRequest;
- }
- if (StringUtils.isEmpty(streamingConfig.getName())) {
- logger.info("StreamingConfig should not be empty.");
- throw new BadRequestException("StremingConfig name should not be empty.");
- }
+ boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
try {
- streamingConfig.setUuid(UUID.randomUUID().toString());
- streamingService.createStreamingConfig(streamingConfig);
-
+ tableDesc.setUuid(UUID.randomUUID().toString());
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ metaMgr.saveSourceTable(tableDesc);
+ cubeMgmtService.syncTableToProject(new String[]{tableDesc.getName()}, project);
} catch (IOException e) {
- logger.error("Failed to save StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to save StreamingConfig: " + e.getLocalizedMessage());
+ throw new BadRequestException("Failed to add streaming table.");
}
+
try {
- kafkaConfig.setUuid(UUID.randomUUID().toString());
- kafkaConfigService.createKafkaConfig(kafkaConfig);
- }catch (IOException e){
+ if (StringUtils.isEmpty(streamingConfig.getName())) {
+ logger.info("StreamingConfig should not be empty.");
+ throw new BadRequestException("StremingConfig name should not be empty.");
+ }
+ try {
+ streamingConfig.setUuid(UUID.randomUUID().toString());
+ streamingService.createStreamingConfig(streamingConfig);
+ saveStreamingSuccess = true;
+ } catch (IOException e) {
+ logger.error("Failed to save StreamingConfig:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to save StreamingConfig: " + e.getLocalizedMessage());
+ }
try {
- streamingService.dropStreamingConfig(streamingConfig);
- } catch (IOException e1) {
- throw new InternalErrorException("StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage());
+ kafkaConfig.setUuid(UUID.randomUUID().toString());
+ kafkaConfigService.createKafkaConfig(kafkaConfig);
+ saveKafkaSuccess = true;
+ } catch (IOException e) {
+ try {
+ streamingService.dropStreamingConfig(streamingConfig);
+ } catch (IOException e1) {
+ throw new InternalErrorException("StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage());
+ }
+ logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage());
}
- logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage());
+ }finally {
+ if(saveKafkaSuccess == false || saveStreamingSuccess == false){
+
+ if(saveStreamingSuccess == true){
+ StreamingConfig sConfig = streamingService.getStreamingManager().getStreamingConfig(streamingConfig.getName());
+ try {
+ streamingService.dropStreamingConfig(sConfig);
+ } catch (IOException e) {
+ throw new InternalErrorException("Action failed and failed to rollback the created streaming config: " + e.getLocalizedMessage());
+ }
+ }
+ if(saveKafkaSuccess == true){
+ try {
+ KafkaConfig kConfig = kafkaConfigService.getKafkaConfig(kafkaConfig.getName());
+ kafkaConfigService.dropKafkaConfig(kConfig);
+ } catch (IOException e) {
+ throw new InternalErrorException("Action failed and failed to rollback the created kafka config: " + e.getLocalizedMessage());
+ }
+ }
+ }
+
}
streamingRequest.setSuccessful(true);
return streamingRequest;
@@ -172,6 +214,24 @@ public class StreamingController extends BasicController {
}
}
+ private TableDesc deserializeTableDesc(StreamingRequest streamingRequest) {
+ TableDesc desc = null;
+ try {
+ logger.debug("Saving TableDesc " + streamingRequest.getTableData());
+ desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
+ } catch (JsonParseException e) {
+ logger.error("The TableDesc definition is not valid.", e);
+ updateRequest(streamingRequest, false, e.getMessage());
+ } catch (JsonMappingException e) {
+ logger.error("The data TableDesc definition is not valid.", e);
+ updateRequest(streamingRequest, false, e.getMessage());
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return desc;
+ }
+
private StreamingConfig deserializeSchemalDesc(StreamingRequest streamingRequest) {
StreamingConfig desc = null;
try {
@@ -222,4 +282,7 @@ public class StreamingController extends BasicController {
this.kafkaConfigService = kafkaConfigService;
}
+ public void setCubeService(CubeService cubeService) {
+ this.cubeMgmtService = cubeService;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/server/src/main/java/org/apache/kylin/rest/request/StreamingRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/StreamingRequest.java b/server/src/main/java/org/apache/kylin/rest/request/StreamingRequest.java
index 07c30f3..a8a983b 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/StreamingRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/StreamingRequest.java
@@ -19,7 +19,8 @@
package org.apache.kylin.rest.request;
-import java.lang.String;public class StreamingRequest {
+import java.lang.String;
+public class StreamingRequest {
private String project;
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/server/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/server/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
index fc663dd..5dede2a 100644
--- a/server/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
+++ b/server/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
@@ -74,6 +74,7 @@ public class TableDescResponse extends TableDesc {
this.setColumns(table.getColumns());
this.setDatabase(table.getDatabase());
this.setName(table.getName());
+ this.setSourceType(table.getSourceType());
this.setUuid(table.getUuid());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 24bdf8d..748019f 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -380,73 +380,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
});
};
- //save streaming
- $scope.postData = {};
- $scope.saveStreaming = function () {
- $scope.kafkaMeta.name = $scope.streamingMeta.name;
-
- $scope.postData.streamingMeta = angular.toJson($scope.streamingMeta, true);
- $scope.postData.kafkaMeta = angular.toJson($scope.kafkaMeta, true);
- SweetAlert.swal({
- title: '',
- text: 'Are you sure to save Streaming ?',
- type: '',
- showCancelButton: true,
- confirmButtonColor: '#DD6B55',
- confirmButtonText: "Yes",
- closeOnConfirm: true
- }, function (isConfirm) {
- if (isConfirm) {
- loadingRequest.show();
-
- if ($scope.modelMode == "editExistStreaming") {
- StreamingService.update({}, {
- project: $scope.projectModel.selectedProject,
- streamingConfig: $scope.postData.streamingMeta,
- kafkaConfig: $scope.postData.kafkaMeta
- }, function (request) {
- if (request.successful) {
- SweetAlert.swal('', 'Updated the streaming successfully.', 'success');
- $location.path("/models");
- } else {
- var message = request.message;
- var msg = !!(message) ? message : 'Failed to take action.';
- MessageService.sendMsg($scope.streamingResultTmpl({
- 'text': msg,
- 'streamingSchema': $scope.postData.streamingMeta,
- 'kfkSchema': $scope.postData.kafkaMeta
- }), 'error', {}, true, 'top_center');
- }
- loadingRequest.hide();
- })
- } else {
- StreamingService.save({}, {
- project: $scope.projectModel.selectedProject,
- streamingConfig: $scope.postData.streamingMeta,
- kafkaConfig: $scope.postData.kafkaMeta
- }, function (request) {
- if (request.successful) {
- SweetAlert.swal('', 'Created the streaming successfully.', 'success');
- $location.path("/models");
- } else {
- var message = request.message;
- var msg = !!(message) ? message : 'Failed to take action.';
- MessageService.sendMsg($scope.streamingResultTmpl({
- 'text': msg,
- 'streamingSchema': $scope.postData.streamingMeta,
- 'kfkSchema': $scope.postData.kafkaMeta
- }), 'error', {}, true, 'top_center');
- }
- loadingRequest.hide();
- })
- }
-
- }
- });
-
-
- }
-
// reverse the date
$scope.saveCubeRollBack = function () {
@@ -732,31 +665,15 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
});
-
- //dimensions options is depend on the model input when add cube
- //$scope.$watch('cubeMetaFrame.model_name', function (newValue, oldValue) {
- // if (!newValue) {
- // return;
- // }
- // $scope.metaModel.model = modelsManager.getModel(newValue);
- //
- // if(!$scope.metaModel.model){
- // return;
- // }
- //
- // var factTable = $scope.metaModel.model.fact_table;
- // var cols = $scope.getColumnsByTable(factTable);
- //
- // for(var i=0;i<cols.length;i++){
- // var col = cols[i];
- // if(col.datatype === "timestamp"){
- // $scope.streamingCfg.columnOptions.push(col.name);
- // }
- // }
- // $scope.kafkaMeta.parserProperties = "tsColName=' ';formatTs=TRUE";
- //
- //
- //});
+ $scope.$watch('cubeMetaFrame.model_name', function (newValue, oldValue) {
+ if (!newValue) {
+ return;
+ }
+ $scope.metaModel.model = modelsManager.getModel(newValue);
+ if(!$scope.metaModel.model){
+ return;
+ }
+ });
$scope.$on('DimensionsEdited', function (event) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/controllers/modelSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelSchema.js b/webapp/app/js/controllers/modelSchema.js
index 3c66609..7885758 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -139,7 +139,7 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
else {
//form validation
if ($scope.forms[$scope.curStep.form].$invalid) {
- $scope.forms[$scope.curStep.form].$sbumitted = true;
+ $scope.forms[$scope.curStep.form].$submitted = true;
return false;
} else {
//business rule check
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index 472b18b..1466a5b 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -422,7 +422,7 @@ KylinApp
});
};
- var StreamingSourceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, scope, tableConfig,cubeConfig,StreamingModel) {
+ var StreamingSourceCtrl = function ($scope, $location,$interpolate,$templateCache, $modalInstance, tableNames, MessageService, projectName, scope, tableConfig,cubeConfig,StreamingModel,StreamingService) {
$scope.cubeState={
"isStreaming": false
@@ -449,19 +449,26 @@ KylinApp
}
$scope.nextStep = function(){
+
+ $scope.checkFailed = false;
+
//check form
- $scope.form['setStreamingSchema'].$sbumitted = true;
+ $scope.form['setStreamingSchema'].$submitted = true;
if(!$scope.streaming.sourceSchema||$scope.streaming.sourceSchema===""){
- return;
+ $scope.checkFailed = true;
}
if(!$scope.table.name||$scope.table.name===""){
- return;
+ $scope.checkFailed = true;
}
$scope.prepareNextStep();
if(!$scope.rule.timestampColumnExist){
+ $scope.checkFailed = true;
+ }
+
+ if($scope.checkFailed){
return;
}
@@ -470,6 +477,7 @@ KylinApp
$scope.prepareNextStep = function(){
$scope.streamingCfg.columnOptions = [];
+ $scope.rule.timestampColumnExist = false;
angular.forEach($scope.columnList,function(column,$index){
if (column.checked == "Y" && column.fromSource=="Y" && column.type == "timestamp") {
$scope.streamingCfg.columnOptions.push(column.name);
@@ -551,9 +559,6 @@ KylinApp
'fromSource':'Y'
});
-
-
- //var formatList = [];
//var
columnList = _.sortBy(columnList, function (i) { return i.type; });
}
@@ -587,15 +592,35 @@ KylinApp
})
}
$scope.columnList = columnList;
-
}
+
+ $scope.streamingResultTmpl = function (notification) {
+ // Get the static notification template.
+ var tmpl = notification.type == 'success' ? 'streamingResultSuccess.html' : 'streamingResultError.html';
+ return $interpolate($templateCache.get(tmpl))(notification);
+ };
+
+
$scope.form={};
$scope.rule={
'timestampColumnExist':false
}
+
+ $scope.modelMode == "addStreaming";
+
$scope.syncStreamingSchema = function () {
+ $scope.form['cube_streaming_form'].$submitted = true;
+
+ if($scope.form['cube_streaming_form'].parserName.$invalid || $scope.form['cube_streaming_form'].parserProperties.$invalid) {
+ $scope.state.isParserHeaderOpen = true;
+ }
+
+ if($scope.form['cube_streaming_form'].$invalid){
+ return;
+ }
+
var columns = [];
angular.forEach($scope.columnList,function(column,$index){
if (column.checked == "Y") {
@@ -616,10 +641,13 @@ KylinApp
'database':'Default'
}
+
+ $scope.kafkaMeta.name = $scope.table.name
+ $scope.streamingMeta.name = $scope.table.name;
+
SweetAlert.swal({
- title: '',
- text: 'Are you sure to create the streaming table info?',
- type: '',
+ title:"",
+ text: 'Are you sure to save the streaming table and cluster info ?',
showCancelButton: true,
confirmButtonColor: '#DD6B55',
confirmButtonText: "Yes",
@@ -627,31 +655,93 @@ KylinApp
}, function (isConfirm) {
if (isConfirm) {
loadingRequest.show();
- TableService.addStreamingSrc({}, {project: $scope.projectName,tableData:angular.toJson($scope.tableData)}, function (request) {
- if(request.success){
+
+ if ($scope.modelMode == "editExistStreaming") {
+ StreamingService.update({}, {
+ project: $scope.projectName,
+ tableData:angular.toJson($scope.tableData),
+ streamingConfig: angular.toJson($scope.streamingMeta),
+ kafkaConfig: angular.toJson($scope.kafkaMeta)
+ }, function (request) {
+ if (request.successful) {
+ SweetAlert.swal('', 'Updated the streaming successfully.', 'success');
+ $location.path("/models");
+ } else {
+ var message = request.message;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema': angular.toJson($scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ }
loadingRequest.hide();
- SweetAlert.swal('', 'Create Streaming Table Schema Successfully.', 'success');
- $scope.cancel();
- scope.aceSrcTbLoaded(true);
- return;
- }else{
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- //end loading
- loadingRequest.hide();
- }, function (e) {
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- loadingRequest.hide();
- });
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema': angular.toJson($scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ } else {
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema': angular.toJson($scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ }
+ //end loading
+ loadingRequest.hide();
+
+ })
+ } else {
+ StreamingService.save({}, {
+ project: $scope.projectName,
+ tableData:angular.toJson($scope.tableData),
+ streamingConfig: angular.toJson($scope.streamingMeta),
+ kafkaConfig: angular.toJson($scope.kafkaMeta)
+ }, function (request) {
+ if (request.successful) {
+ SweetAlert.swal('', 'Created the streaming successfully.', 'success');
+ location.reload();
+ } else {
+ var message = request.message;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema': angular.toJson($scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ }
+ loadingRequest.hide();
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema':angular.toJson( $scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ } else {
+ MessageService.sendMsg($scope.streamingResultTmpl({
+ 'text': msg,
+ 'streamingSchema': angular.toJson($scope.streamingMeta,true),
+ 'kfkSchema': angular.toJson($scope.kafkaMeta,true)
+ }), 'error', {}, true, 'top_center');
+ }
+ //end loading
+ loadingRequest.hide();
+ })
+ }
+
}
});
}
+
};
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/controllers/streamingConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/streamingConfig.js b/webapp/app/js/controllers/streamingConfig.js
index b0a1ebd..4e916db 100644
--- a/webapp/app/js/controllers/streamingConfig.js
+++ b/webapp/app/js/controllers/streamingConfig.js
@@ -18,7 +18,13 @@
'use strict';
-KylinApp.controller('streamingConfigCtrl', function ($scope, $q, $routeParams, $location, $window, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, modelConfig, ProjectModel, ModelService, MetaModel, modelsManager, cubesManager, TableModel, $animate,StreamingService,StreamingModel) {
+KylinApp.controller('streamingConfigCtrl', function ($scope,StreamingService, $q, $routeParams, $location, $window, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, modelConfig, ProjectModel, ModelService, MetaModel, modelsManager, cubesManager, TableModel, $animate,StreamingModel) {
+
+ $scope.tableModel = TableModel;
+ $scope.streamingMeta = StreamingModel.createStreamingConfig();
+ $scope.kafkaMeta = StreamingModel.createKafkaConfig();
+
+
$scope.addCluster = function () {
$scope.kafkaMeta.clusters.push(StreamingModel.createKafkaCluster());
@@ -80,4 +86,27 @@ KylinApp.controller('streamingConfigCtrl', function ($scope, $q, $routeParams, $
$scope.kafkaMeta.parserProperties = "tsColName="+$scope.streamingCfg.parseTsColumn;
}
+ $scope.$watch('tableModel.selectedSrcTable', function (newValue, oldValue) {
+ if (!newValue) {
+ return;
+ }
+ //view model
+ if($scope.state.mode == 'view' && $scope.tableModel.selectedSrcTable.source_type==1){
+ var table = $scope.tableModel.selectedSrcTable;
+ StreamingService.getConfig({table:table.name}, function (configs) {
+ if(!!configs[0]&&configs[0].name.toUpperCase() == table.name.toUpperCase()){
+ $scope.streamingMeta = configs[0];
+ StreamingService.getKfkConfig({kafkaConfigName:$scope.streamingMeta.name}, function (streamings) {
+ if(!!streamings[0]&&streamings[0].name.toUpperCase() == table.name.toUpperCase()){
+ $scope.kafkaMeta = streamings[0];
+ }
+ })
+ }
+ })
+ }
+
+
+
+ });
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/model/streamingModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/streamingModel.js b/webapp/app/js/model/streamingModel.js
index 61560d7..0321806 100644
--- a/webapp/app/js/model/streamingModel.js
+++ b/webapp/app/js/model/streamingModel.js
@@ -22,8 +22,7 @@ KylinApp.service('StreamingModel', function () {
this.createStreamingConfig = function () {
var streamingConfig = {
"name": "",
- "iiName": "",
- "cubeName": ""
+ "type":"kafka"
};
return streamingConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/js/model/tableModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/tableModel.js b/webapp/app/js/model/tableModel.js
index bb124c9..788fbae 100755
--- a/webapp/app/js/model/tableModel.js
+++ b/webapp/app/js/model/tableModel.js
@@ -106,10 +106,15 @@ KylinApp.service('TableModel', function(ProjectModel,$q,TableService,$log) {
var _table_node_list = [];
angular.forEach(tables,function(_table){
+ var table_icon="fa fa-table";
+ if(_table.source_type==1){
+ table_icon="fa fa-th"
+ }
+
var _table_node = {
label:_table.name,
data:_table,
- icon:"fa fa-table",
+ icon:table_icon,
onSelect:function(branch){
// set selected model
_this.selectedSrcTable = branch.data;
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/cubeDesigner/info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/info.html b/webapp/app/partials/cubeDesigner/info.html
index c9fee9b..a1d876b 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -25,7 +25,7 @@
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
<b>Model Name</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.model_name.$invalid && (forms.cube_info_form.model_name.$dirty||forms.cube_info_form.model_name.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.model_name.$invalid && (forms.cube_info_form.model_name.$dirty||forms.cube_info_form.model_name.$submitted)}">
<select chosen ng-model="cubeMetaFrame.model_name" ng-if="cubeMode=='addNewCube'"
ng-options="model for model in modelsManager.modelNameList"
style="width:100%;"
@@ -39,7 +39,7 @@
<input ng-model="cubeMetaFrame.model_name" ng-disabled="true" class="form-control" ng-if="cubeMode=='editExistCube'"/>
- <small class="help-block" ng-show="forms.cube_info_form.model_name.$error.required && (forms.cube_info_form.model_name.$dirty||forms.cube_info_form.$sbumitted)">Model name is required.</small>
+ <small class="help-block" ng-show="forms.cube_info_form.model_name.$error.required && (forms.cube_info_form.model_name.$dirty||forms.cube_info_form.$submitted)">Model name is required.</small>
<span ng-if="state.mode=='view'">{{cubeMetaFrame.model_name}}</span>
</div>
@@ -54,13 +54,13 @@
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
<b>Cube Name</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.cube_name.$invalid && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.cube_name.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.cube_name.$invalid && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.cube_name.$submitted)}">
<input ng-if="state.mode=='edit'" ng-disabled="{{isEdit}}" name="cube_name" type="text" class="form-control"
ng-model="cubeMetaFrame.name" required
placeholder="You can use letters, numbers, and '_'"
ng-maxlength=100 ng-pattern="/^\w+$/" />
- <small class="help-block" ng-show="forms.cube_info_form.cube_name.$error.required && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.$sbumitted)">Cube name is required.</small>
- <small class="help-block" ng-show="!forms.cube_info_form.cube_name.$error.required&&forms.cube_info_form.cube_name.$invalid && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.$sbumitted)">Cube name is invalid.</small>
+ <small class="help-block" ng-show="forms.cube_info_form.cube_name.$error.required && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.$submitted)">Cube name is required.</small>
+ <small class="help-block" ng-show="!forms.cube_info_form.cube_name.$error.required&&forms.cube_info_form.cube_name.$invalid && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.$submitted)">Cube name is invalid.</small>
<span ng-if="state.mode=='view'">{{cubeMetaFrame.name}}</span>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/cubeDesigner/kafkaBasicConfig.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/kafkaBasicConfig.html b/webapp/app/partials/cubeDesigner/kafkaBasicConfig.html
index d9ca5b1..67c0427 100644
--- a/webapp/app/partials/cubeDesigner/kafkaBasicConfig.html
+++ b/webapp/app/partials/cubeDesigner/kafkaBasicConfig.html
@@ -23,11 +23,11 @@
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
<b>Topic</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.kafka_config_form.topic.$invalid && (forms.kafka_config_form.topic.$dirty||forms.kafka_config_form.topic.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.kafka_config_form.topic.$invalid && (forms.kafka_config_form.topic.$dirty||forms.kafka_config_form.topic.$submitted)}">
<input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.topic" type="text"
placeholder="Input kafkaConfig topic"
class="form-control"/>
- <small class="help-block" ng-show="forms.kafka_config_form.topic.$error.required && (forms.kafka_config_form.topic.$dirty||forms.kafka_config_form.$sbumitted)">Kafka topic is required.</small>
+ <small class="help-block" ng-show="forms.kafka_config_form.topic.$error.required && (forms.kafka_config_form.topic.$dirty||forms.kafka_config_form.$submitted)">Kafka topic is required.</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.topic}}</span>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/cubeDesigner/streamingConfig.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/streamingConfig.html b/webapp/app/partials/cubeDesigner/streamingConfig.html
index c3afd26..38cb2cd 100644
--- a/webapp/app/partials/cubeDesigner/streamingConfig.html
+++ b/webapp/app/partials/cubeDesigner/streamingConfig.html
@@ -17,7 +17,7 @@
-->
<div ng-controller="streamingConfigCtrl">
- <ng-form name="forms.cube_streaming_form" novalidate>
+ <form name="form.cube_streaming_form" novalidate>
<div>
<accordion>
@@ -33,11 +33,15 @@
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
<b>Topic</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_streaming_form.topic.$invalid && (forms.cube_streaming_form.topic.$dirty||forms.cube_streaming_form.topic.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':form.cube_streaming_form.topic.$invalid && (form.cube_streaming_form.topic.$dirty||form.cube_streaming_form.$submitted)}">
<input ng-if="state.mode=='edit'" name="topic" required ng-model="kafkaMeta.topic" type="text"
placeholder="Input kafkaConfig topic"
class="form-control"/>
- <small class="help-block" ng-show="forms.cube_streaming_form.topic.$error.required && (forms.cube_streaming_form.topic.$dirty||forms.cube_streaming_form.$sbumitted)">Kafka topic is required.</small>
+ <small class="help-block"
+ ng-show="!form.cube_streaming_form.topic.$error.required && form.cube_streaming_form.topic.$invalid && (form.cube_streaming_form.topic.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka topic is invalid.
+ </small>
+ <small class="help-block" ng-show="form.cube_streaming_form.topic.$error.required && (form.cube_streaming_form.topic.$dirty||form.cube_streaming_form.$submitted)">Kafka topic is required.</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.topic}}</span>
</div>
</div>
@@ -136,12 +140,17 @@
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':forms.cube_streaming_form.timeout.$invalid && (forms.cube_streaming_form.timeout.$dirty||forms.cube_streaming_form.timeout.$sbumitted)}">
- <input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.timeout" type="text"
+ ng-class="{'has-error':form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="timeout" required ng-model="kafkaMeta.timeout" type="text"
placeholder="Input kafkaConfig timeout"
+ ng-maxlength=100 ng-pattern="/^\+?[1-9][0-9]*$/"
class="form-control"/>
<small class="help-block"
- ng-show="forms.cube_streaming_form.timeout.$error.required && (forms.cube_streaming_form.timeout.$dirty||forms.cube_streaming_form.$sbumitted)">
+ ng-show="!form.cube_streaming_form.timeout.$error.required && form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka timeout is invalid.
+ </small>
+ <small class="help-block"
+ ng-show="form.cube_streaming_form.timeout.$error.required && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
Kafka timeout is required.
</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.timeout}}</span>
@@ -156,12 +165,17 @@
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':forms.cube_streaming_form.bufferSize.$invalid && (forms.cube_streaming_form.bufferSize.$dirty||forms.cube_streaming_form.bufferSize.$sbumitted)}">
- <input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.bufferSize" type="text"
+ ng-class="{'has-error':form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="bufferSize" required ng-model="kafkaMeta.bufferSize" type="text"
placeholder="Input kafkaConfig bufferSize"
+ ng-maxlength=100 ng-pattern="/^\+?[1-9][0-9]*$/"
class="form-control"/>
<small class="help-block"
- ng-show="forms.cube_streaming_form.bufferSize.$error.required && (forms.cube_streaming_form.bufferSize.$dirty||forms.cube_streaming_form.$sbumitted)">
+ ng-show="!form.cube_streaming_form.bufferSize.$error.required && form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka bufferSize is invalid.
+ </small>
+ <small class="help-block"
+ ng-show="form.cube_streaming_form.bufferSize.$error.required && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
Kafka bufferSize is required.
</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.bufferSize}}</span>
@@ -176,12 +190,17 @@
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':forms.cube_streaming_form.margin.$invalid && (forms.cube_streaming_form.margin.$dirty||forms.cube_streaming_form.margin.$sbumitted)}">
- <input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.margin" type="text"
+ ng-class="{'has-error':form.cube_streaming_form.margin.$invalid && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="margin" required ng-model="kafkaMeta.margin" type="text"
placeholder="Input kafkaConfig margin"
+ ng-maxlength=100 ng-pattern="/^\+?[1-9][0-9]*$/"
class="form-control"/>
<small class="help-block"
- ng-show="forms.cube_streaming_form.margin.$error.required && (forms.cube_streaming_form.margin.$dirty||forms.cube_streaming_form.$sbumitted)">
+ ng-show="!form.cube_streaming_form.margin.$error.required && form.cube_streaming_form.margin.$invalid && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka margin is invalid.
+ </small>
+ <small class="help-block"
+ ng-show="form.cube_streaming_form.margin.$error.required && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)">
Kafka margin is required.
</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.margin}}</span>
@@ -208,13 +227,17 @@
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':forms.cube_streaming_form.parserName.$invalid && (forms.cube_streaming_form.parserName.$dirty||forms.cube_streaming_form.parserName.$sbumitted)}">
- <input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.parserName" type="text"
+ ng-class="{'has-error':form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="parserName" required ng-model="kafkaMeta.parserName" type="text"
placeholder="Input kafkaConfig parserName"
class="form-control"/>
<small class="help-block"
- ng-show="forms.cube_streaming_form.parserName.$error.required && (forms.cube_streaming_form.parserName.$dirty||forms.cube_streaming_form.$sbumitted)">
- Kafka parserName is required.
+ ng-show="!form.cube_streaming_form.parserName.$error.required && form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka parser name is invalid.
+ </small>
+ <small class="help-block"
+ ng-show="form.cube_streaming_form.parserName.$error.required && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka parser name is required.
</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.parserName}}</span>
</div>
@@ -244,13 +267,17 @@
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':forms.cube_streaming_form.parserProperties.$invalid && (forms.cube_streaming_form.parserProperties.$dirty||forms.cube_streaming_form.parserProperties.$sbumitted)}">
- <input ng-if="state.mode=='edit'" name="name" required ng-model="kafkaMeta.parserProperties" type="text"
+ ng-class="{'has-error':form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="parserProperties" required ng-model="kafkaMeta.parserProperties" type="text"
placeholder="configA=1;configB=2"
class="form-control"/>
<small class="help-block"
- ng-show="forms.cube_streaming_form.parserProperties.$error.required && (forms.cube_streaming_form.parserProperties.$dirty||forms.cube_streaming_form.$sbumitted)">
- Parser Properties is required.
+ ng-show="!form.cube_streaming_form.parserProperties.$error.required && form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+ Parser properties is invalid.
+ </small>
+ <small class="help-block"
+ ng-show="form.cube_streaming_form.parserProperties.$error.required && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+ Parser properties is required.
</small>
<span ng-if="state.mode=='view'">{{kafkaMeta.parserProperties}}</span>
</div>
@@ -260,5 +287,5 @@
</accordion>
</div>
- </ng-form>
+ </form>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 83ba101..d1a9cdd 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -25,7 +25,7 @@
<label class="col-xs-12 col-sm-2 control-label concube.detailtrol-label no-padding-right font-color-default">
<b>Fact Table</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$submitted)}">
<select chosen ng-model="modelsManager.selectedModel.fact_table" ng-if="state.mode=='edit'"
ng-options="table.name as table.name for table in tableModel.selectProjectTables"
style="width:100%;"
@@ -36,7 +36,7 @@
<option value=""> -- Select Fact Table -- </option>
</select>
- <small class="help-block" ng-show="forms.data_model_form.innerform.typeahead.$error.required && (forms.data_model_form.innerform.typeahead.$dirty||forms.data_model_form.$sbumitted)">The fact table is required</small>
+ <small class="help-block" ng-show="forms.data_model_form.innerform.typeahead.$error.required && (forms.data_model_form.innerform.typeahead.$dirty||forms.data_model_form.$submitted)">The fact table is required</small>
<span ng-if="state.mode=='view'">{{modelsManager.selectedModel.fact_table}}</span>
</div>
</div>
@@ -121,7 +121,7 @@
<div class="form-group">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Lookup Table Name</b></label>
- <div ng-class="{'has-error':lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$sbumitted)}">
+ <div ng-class="{'has-error':lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)}">
<div class="col-xs-12 col-sm-6">
<select chosen ng-model="newLookup.table"
ng-options="table.name as table.name for table in tableModel.selectProjectTables"
@@ -133,7 +133,7 @@
<option value=""> -- Select Lookup Table -- </option>
</select>
- <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$sbumitted)">Table name required</small>
+ <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/modelDesigner/model_info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_info.html b/webapp/app/partials/modelDesigner/model_info.html
index bace322..a6ad300 100644
--- a/webapp/app/partials/modelDesigner/model_info.html
+++ b/webapp/app/partials/modelDesigner/model_info.html
@@ -28,15 +28,15 @@
<b>Model Name</b>
</label>
- <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$sbumitted)}">
+ <div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)}">
<!-- edit -->
<input ng-if="state.mode=='edit'" name="model_name" type="text" class="form-control" ng-disabled="{{modelMode=='editExistModel'}}"
ng-model="modelsManager.selectedModel.name" required
placeholder="You can use letters, numbers, and '_'"
ng-maxlength=100 ng-pattern="/^\w+$/" />
- <small class="help-block" ng-show="forms.model_info_form.model_name.$error.required && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$sbumitted)">Model name is required.</small>
- <small class="help-block" ng-show="!forms.model_info_form.model_name.$error.required&&forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$sbumitted)">Model name is invalid.</small>
+ <small class="help-block" ng-show="forms.model_info_form.model_name.$error.required && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)">Model name is required.</small>
+ <small class="help-block" ng-show="!forms.model_info_form.model_name.$error.required&&forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)">Model name is invalid.</small>
<span ng-if="state.mode=='view'">{{modelsManager.selectedModel.name}}</span>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/tables/loadStreamingTable.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/loadStreamingTable.html b/webapp/app/partials/tables/loadStreamingTable.html
index 5e6116c..da79369 100644
--- a/webapp/app/partials/tables/loadStreamingTable.html
+++ b/webapp/app/partials/tables/loadStreamingTable.html
@@ -5,19 +5,20 @@
Need to input streaming source record here, will detect the source schema and create a table schema for
streaming.
</p>
- <div class="has-error" ng-if="!table.sourceValid&&table.schemaChecked">
- <small class="help-block">
- Source json invalid.
- </small>
- </div>
+
<div style="padding:15px;" class="has-error">
- <small class="help-block" ng-show="streaming.sourceSchema==''&&form.setStreamingSchema.$sbumitted">Please
+ <small class="help-block" ng-show="streaming.sourceSchema==''&&form.setStreamingSchema.$submitted">Please
input Streaming source record to generate schema.
</small>
+ <small class="help-block" ng-if="!table.sourceValid&&table.schemaChecked">
+ Source json invalid, Please correct your schema and generate again.
+ </small>
</div>
+
<div style="margin-bottom: 20px;">
<span class="label label-info">JSON</span>
</div>
+
<div ng-model="streaming.sourceSchema" ui-ace="{
useWrapMode : true,
mode:'json',
@@ -40,10 +41,10 @@
<label class="col-xs-4 control-label" style="text-align: left;">Table Name</label>
<div class="col-xs-8"
- ng-class="{'has-error':form.setStreamingSchema.streamingObject.$invalid && (form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)}">
+ ng-class="{'has-error':form.setStreamingSchema.streamingObject.$invalid && (form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$submitted)}">
<input type="text" name="streamingObject" required="" ng-model="table.name" class="form-control"/>
<small class="help-block"
- ng-show="form.setStreamingSchema.streamingObject.$error.required&&(form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)">
+ ng-show="form.setStreamingSchema.streamingObject.$error.required&&(form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$submitted)">
Table name is required.
</small>
</div>
@@ -80,7 +81,7 @@
</tr>
</table>
- <div class="has-error" ng-if="!rule.timestampColumnExist&&form.setStreamingSchema.$sbumitted">
+ <div class="has-error" ng-if="!rule.timestampColumnExist&&form.setStreamingSchema.$submitted">
<small class="help-block">
You should choose at least one 'timestamp' type column generated from source schema.
</small>
http://git-wip-us.apache.org/repos/asf/kylin/blob/376d695c/webapp/app/partials/tables/table_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/table_detail.html b/webapp/app/partials/tables/table_detail.html
index b232375..a4548a1 100644
--- a/webapp/app/partials/tables/table_detail.html
+++ b/webapp/app/partials/tables/table_detail.html
@@ -28,6 +28,9 @@
<li>
<a data-toggle="tab" href="#schema">Extend Information</a>
</li>
+ <li>
+ <a data-toggle="tab" ng-if="tableModel.selectedSrcTable.source_type==1" href="#streaming">Streaming Cluster</a>
+ </li>
</ul>
<div class="tab-content">
<!--Schema-->
@@ -143,8 +146,15 @@
</div>
</div>
</div>
+
+ <!--streaming-->
+ <div id="streaming" ng-show="tableModel.selectedSrcTable.source_type==1" class="tab-pane">
+ <div ng-include="'partials/cubeDesigner/streamingConfig.html'" ng-init="state.mode='view'"></div>
+ </div>
+
+ </div>
+
</div>
- </div>
</div>
[47/50] [abbrv] kylin git commit: [maven-release-plugin] prepare
release kylin-1.5.0
Posted by sh...@apache.org.
[maven-release-plugin] prepare release kylin-1.5.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d02018
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d02018
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d02018
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 64d02018fcf082745d4104d604646058fcb7e17c
Parents: 4e5a936
Author: Yang Li <li...@apache.org>
Authored: Sat Mar 12 12:54:25 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Mar 12 12:54:25 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 6 ++----
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 5 ++---
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 6 ++----
engine-streaming/pom.xml | 6 ++----
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 5 ++---
kylin-it/pom.xml | 16 +++++++---------
pom.xml | 7 +++----
query/pom.xml | 5 ++---
server/pom.xml | 5 ++---
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 6 ++----
storage-hbase/pom.xml | 8 +++-----
20 files changed, 38 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 16b647e..20f02a0 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,13 +17,11 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 93f52b7..6ca1eae 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 133b6ff..776ba76 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-core-common</artifactId>
@@ -28,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 9fde78a..97af425 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 9718b67..d518ecf 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index ffe09b0..a17418c 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index caf5246..89d85f2 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bb78ac..b529488 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 0d0f7b8..088641b 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 1fdbded..9cce44a 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -17,14 +17,12 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index ab58c59..ee1f498 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -17,9 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-engine-streaming</artifactId>
@@ -29,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 5007629..2082329 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 805f34f..c1311db 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-jdbc</artifactId>
<packaging>jar</packaging>
@@ -28,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index e3fbc84..5b42622 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -18,13 +18,11 @@
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -32,8 +30,8 @@
<name>Kylin:IT</name>
<properties>
- <hdp.version/>
- <fastBuildMode/>
+ <hdp.version />
+ <fastBuildMode />
</properties>
<!-- Dependencies. -->
@@ -305,7 +303,7 @@
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
<argument>-classpath</argument>
- <classpath/>
+ <classpath />
<argument>org.apache.kylin.provision.BuildCubeWithEngine</argument>
</arguments>
<workingDirectory>${project.basedir}</workingDirectory>
@@ -326,7 +324,7 @@
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
<argument>-classpath</argument>
- <classpath/>
+ <classpath />
<argument>org.apache.kylin.provision.BuildCubeWithStream</argument>
</arguments>
<workingDirectory>${project.basedir}</workingDirectory>
@@ -347,7 +345,7 @@
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
<argument>-classpath</argument>
- <classpath/>
+ <classpath />
<argument>org.apache.kylin.provision.BuildIIWithStream</argument>
</arguments>
<workingDirectory>${project.basedir}</workingDirectory>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9caa29c3..f8d95b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -30,7 +29,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
<name>Kylin:HadoopOLAPEngine</name>
<url>http://kylin.apache.org</url>
@@ -168,7 +167,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/kylin.git</url>
- <tag>apache-kylin-2.1</tag>
+ <tag>kylin-1.5.0</tag>
</scm>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 55add61..c1b3213 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<!-- The basics. -->
<artifactId>kylin-query</artifactId>
@@ -29,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index b9ea034..253ce69 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-server</artifactId>
@@ -29,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 46c016d..7a0c560 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 7a8d7b0..4ed14e5 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -17,9 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-source-kafka</artifactId>
@@ -29,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d02018/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index c274874..c15d617 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-storage-hbase</artifactId>
@@ -28,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0</version>
</parent>
@@ -128,8 +127,7 @@
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>coprocessor</shadedClassifierName>
<transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
</transformers>
<artifactSet>
<includes>
[11/50] [abbrv] kylin git commit: KYLIN-1420 Query returns empty
result on partition column's boundary condition
Posted by sh...@apache.org.
KYLIN-1420 Query returns empty result on partition column's boundary condition
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3736f72c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3736f72c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3736f72c
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 3736f72cc756af28a39a473efc47bec943ba7fc9
Parents: 3af7d4a
Author: shaofengshi <sh...@apache.org>
Authored: Mon Feb 15 15:52:15 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:14:18 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3736f72c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index a72426d..d314dde 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -108,8 +108,8 @@ public class GTScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (partitionColRef != null && range.column.equals(partitionColRef)) {
if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 //
- && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0) {
- //segment range is [Closed,Open)
+ && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) <= 0) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <=.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}",//
new Object[] { partitionColRef, makeReadable(segmentStartAndEnd.getFirst()), makeReadable(segmentStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
[34/50] [abbrv] kylin git commit: minor,
revise InMemCubeBuilderTest to cover both SSB and orignal kylin test
cube
Posted by sh...@apache.org.
minor, revise InMemCubeBuilderTest to cover both SSB and orignal kylin test cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4662adab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4662adab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4662adab
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 4662adab73fb87591400afa11b6741198d9fa7e4
Parents: 18d4a5e
Author: Li Yang <li...@apache.org>
Authored: Fri Mar 11 13:48:09 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Mar 11 13:49:22 2016 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/InMemCubeBuilderTest.java | 64 +++++++++++++-------
1 file changed, 41 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4662adab/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index 76674d3..d96de4f 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -46,8 +46,8 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,36 +61,54 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
private static final Logger logger = LoggerFactory.getLogger(InMemCubeBuilderTest.class);
- private static final int INPUT_ROWS = 1000;
- private static final int THREADS = 1;
+ private CubeInstance cube;
+ private String flatTable;
+ private Map<TblColRef, Dictionary<String>> dictionaryMap;
- private static CubeInstance cube;
- private static String flatTable;
- private static Map<TblColRef, Dictionary<String>> dictionaryMap;
+ private int nInpRows;
+ private int nThreads;
- @BeforeClass
- public static void before() throws IOException {
- staticCreateTestMetadata();
+ @Before
+ public void before() throws IOException {
+ createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testKylinCube() throws Exception {
+ testBuild("test_kylin_cube_without_slr_left_join_empty", //
+ "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv", 70000, 4);
+ }
+ @Test
+ public void testSSBCube() throws Exception {
+ testBuild("ssb", //
+ "../examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv", 1000, 1);
+ }
+
+ public void testBuild(String cubeName, String flatTable, int nInpRows, int nThreads) throws Exception {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
- cube = cubeManager.getCube("ssb");
- flatTable = "../examples/test_case_data/localmeta/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv";
- dictionaryMap = getDictionaryMap(cube, flatTable);
- }
+ this.nInpRows = nInpRows;
+ this.nThreads = nThreads;
+
+ this.cube = cubeManager.getCube(cubeName);
+ this.flatTable = flatTable;
+ this.dictionaryMap = getDictionaryMap(cube, flatTable);
- @AfterClass
- public static void after() throws Exception {
- staticCleanupTestMetadata();
+ testBuildInner();
}
- @Test
- public void test() throws Exception {
+ private void testBuildInner() throws Exception {
InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(cube.getDescriptor(), dictionaryMap);
//DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
- cubeBuilder.setConcurrentThreads(THREADS);
+ cubeBuilder.setConcurrentThreads(nThreads);
ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -99,7 +117,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
// round 1
{
Future<?> future = executorService.submit(cubeBuilder.buildAsRunnable(queue, new ConsoleGTRecordWriter()));
- feedData(cube, flatTable, queue, INPUT_ROWS);
+ feedData(cube, flatTable, queue, nInpRows);
future.get();
}
@@ -113,7 +131,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
// round 3
{
Future<?> future = executorService.submit(cubeBuilder.buildAsRunnable(queue, new ConsoleGTRecordWriter()));
- feedData(cube, flatTable, queue, INPUT_ROWS);
+ feedData(cube, flatTable, queue, nInpRows);
future.get();
}
@@ -213,7 +231,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
List<String> lines = FileUtils.readLines(new File(flatTable), "UTF-8");
for (String line : lines) {
String[] row = line.trim().split(",");
- if ( row.length != nColumns) {
+ if (row.length != nColumns) {
throw new IllegalStateException();
}
if (row[c] != null) {
[06/50] [abbrv] kylin git commit: KYLIN-1465 log improvement on others
Posted by sh...@apache.org.
KYLIN-1465 log improvement on others
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a3418b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a3418b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a3418b
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: b9a3418b7f2121745090bb6463e7be754babfb73
Parents: b41c446
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 4 15:59:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 4 16:01:04 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeGridTable.java | 6 ++---
.../gridtable/NotEnoughGTInfoException.java | 23 --------------------
.../kylin/gridtable/GTScanReqSerDerTest.java | 11 +++++-----
.../kylin/storage/translate/HBaseKeyRange.java | 4 +---
.../streaming/cube/StreamingCubeBuilder.java | 6 +++--
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 14 ++++++------
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 18 +++++++--------
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 8 +++----
.../hbase/cube/v2/CubeSegmentScanner.java | 3 +--
.../storage/hbase/cube/v2/CubeStorageQuery.java | 10 +++------
.../kylin/storage/hbase/cube/v2/RawScan.java | 2 +-
12 files changed, 41 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 05fc8a5..5f0bb07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -29,18 +29,18 @@ public class CubeGridTable {
dictionaryMap.put(col, dictionary);
}
}
-
+
return dictionaryMap;
}
- public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException {
+ public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
for (TblColRef dim : cuboid.getColumns()) {
if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
Dictionary dict = dictionaryMap.get(dim);
if (dict == null) {
- throw new NotEnoughGTInfoException();
+ throw new RuntimeException("Dictionary for " + dim + " is not found");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
deleted file mode 100644
index 9bbcf75..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-@SuppressWarnings("serial")
-public class NotEnoughGTInfoException extends Exception {
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
index 6642d95..77cc2d8 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
@@ -29,7 +29,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -73,7 +72,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
buffer.flip();
GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ this.compareTwoGTInfo(info, sInfo);
}
@Test
@@ -83,11 +82,11 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
buffer.flip();
GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ this.compareTwoGTInfo(info, sInfo);
}
@Test
- public void testGTInfo() throws NotEnoughGTInfoException {
+ public void testGTInfo() {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
CubeSegment segment = cube.getFirstSegment();
@@ -95,8 +94,8 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
GTInfo.serializer.serialize(info, buffer);
buffer.flip();
- GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
+ this.compareTwoGTInfo(info, sInfo);
}
private void compareTwoGTInfo(GTInfo info, GTInfo sInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index a5ca62b..4f39cf6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -45,9 +45,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
- *
* @author xjiang
- *
*/
public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
@@ -147,7 +145,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
- buf.append(System.lineSeparator());
+ buf.append(";");
}
this.fuzzyKeyString = buf.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index d7056cf..c4f2b7e 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -73,6 +73,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
private static final Logger logger = LoggerFactory.getLogger(StreamingCubeBuilder.class);
private final String cubeName;
+ private int processedRowCount = 0;
public StreamingCubeBuilder(String cubeName) {
this.cubeName = cubeName;
@@ -81,16 +82,16 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
@Override
public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter) {
try {
-
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>();
InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap);
final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, cuboidWriter));
+ processedRowCount = streamingBatch.getMessages().size();
for (StreamingMessage streamingMessage : streamingBatch.getMessages()) {
blockingQueue.put(streamingMessage.getData());
}
- blockingQueue.put(Collections.<String> emptyList());
+ blockingQueue.put(Collections.<String>emptyList());
future.get();
cuboidWriter.flush();
@@ -157,6 +158,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
public void commit(IBuildable buildable) {
CubeSegment cubeSegment = (CubeSegment) buildable;
cubeSegment.setStatus(SegmentStatusEnum.READY);
+ cubeSegment.setInputRecords(processedRowCount);
CubeUpdate cubeBuilder = new CubeUpdate(cubeSegment.getCubeInstance());
cubeBuilder.setToAddSegs(cubeSegment);
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 5e842f7..909de39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -210,26 +210,26 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private void logScan(HBaseKeyRange keyRange) {
StringBuilder info = new StringBuilder();
- info.append("\nScan hbase table ").append(tableName).append(": ");
+ info.append(" Scan hbase table ").append(tableName).append(": ");
if (keyRange.getCuboid().requirePostAggregation()) {
- info.append("cuboid require post aggregation, from ");
+ info.append(" cuboid require post aggregation, from ");
} else {
- info.append("cuboid exact match, from ");
+ info.append(" cuboid exact match, from ");
}
info.append(keyRange.getCuboid().getInputID());
info.append(" to ");
info.append(keyRange.getCuboid().getId());
- info.append("\nStart: ");
+ info.append(" Start: ");
info.append(keyRange.getStartKeyAsString());
info.append(" - ");
info.append(Bytes.toStringBinary(keyRange.getStartKey()));
- info.append("\nStop: ");
+ info.append(" Stop: ");
info.append(keyRange.getStopKeyAsString());
info.append(" - ");
info.append(Bytes.toStringBinary(keyRange.getStopKey()));
if (this.scan.getFilter() != null) {
- info.append("\nFuzzy key counts: " + keyRange.getFuzzyKeys().size());
- info.append("\nFuzzy: ");
+ info.append(" Fuzzy key counts: " + keyRange.getFuzzyKeys().size());
+ info.append(" Fuzzy: ");
info.append(keyRange.getFuzzyKeyAsString());
}
logger.info(info.toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 1b8b586..3d7f620 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -493,7 +493,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<Collection<ColumnValueRange>> result = Lists.newArrayList();
if (flatFilter == null) {
- result.add(Collections.<ColumnValueRange> emptyList());
+ result.add(Collections.<ColumnValueRange>emptyList());
return result;
}
@@ -535,7 +535,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
if (globalAlwaysTrue) {
orAndRanges.clear();
- orAndRanges.add(Collections.<ColumnValueRange> emptyList());
+ orAndRanges.add(Collections.<ColumnValueRange>emptyList());
}
return orAndRanges;
}
@@ -762,7 +762,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private void setLimit(TupleFilter filter, StorageContext context) {
boolean goodAggr = context.isExactAggregation();
boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
- boolean goodSort = context.hasSort() == false;
+ boolean goodSort = !context.hasSort();
if (goodAggr && goodFilter && goodSort) {
logger.info("Enable limit " + context.getLimit());
context.enableLimit();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index f22964f..e6f9ac1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -293,15 +293,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicInteger totalScannedCount = new AtomicInteger(0);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size() * shardNum);
+ final String currentThreadName = Thread.currentThread().getName();
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
@Override
public void run() {
for (int i = 0; i < scanRequests.size(); ++i) {
- int scanIndex = i;
CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
- builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex));
+ builder.setGtScanRequest(scanRequestByteStrings.get(i)).setHbaseRawScan(rawScanByteStrings.get(i));
for (IntList intList : hbaseColumnsToGTIntList) {
builder.addHbaseColumnsToGT(intList);
}
@@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
for (Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result : results.entrySet()) {
totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
- logger.info(getStatsString(result));
+ logger.info("<spawned by " + currentThreadName + ">" + getStatsString(result));
try {
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows())));
} catch (IOException | DataFormatException e) {
@@ -335,12 +335,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private String getStatsString(Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result) {
StringBuilder sb = new StringBuilder();
Stats stats = result.getValue().getStats();
- sb.append("Endpoint RPC returned from HTable " + cubeSeg.getStorageLocationIdentifier() + " Shard " + BytesUtil.toHex(result.getKey()) + " on host: " + stats.getHostname() + ".");
- sb.append("Total scanned row: " + stats.getScannedRowCount() + ". ");
- sb.append("Total filtered/aggred row: " + stats.getAggregatedRowCount() + ". ");
- sb.append("Time elapsed in EP: " + (stats.getServiceEndTime() - stats.getServiceStartTime()) + "(ms). ");
- sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize() + ".");
- sb.append("Etc message: " + stats.getEtcMsg() + ".");
+ sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append(" Shard ").append(BytesUtil.toHex(result.getKey())).append(" on host: ").append(stats.getHostname()).append(".");
+ sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". ");
+ sb.append("Total filtered/aggred row: ").append(stats.getAggregatedRowCount()).append(". ");
+ sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime()).append("(ms). ");
+ sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server physical mem left: ").append(stats.getFreePhysicalMemorySize()).append(", server swap mem left:").append(stats.getFreeSwapSpaceSize()).append(".");
+ sb.append("Etc message: ").append(stats.getEtcMsg()).append(".");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index db39455..eb5ac9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -273,20 +273,20 @@ public abstract class CubeHBaseRPC {
info.append(cuboid.getInputID());
info.append(" to ");
info.append(cuboid.getId());
- info.append("\nStart: ");
+ info.append(" Start: ");
info.append(rawScan.getStartKeyAsString());
info.append(" (");
info.append(Bytes.toStringBinary(rawScan.startKey) + ")");
- info.append("\nStop: ");
+ info.append(" Stop: ");
info.append(rawScan.getEndKeyAsString());
info.append(" (");
info.append(Bytes.toStringBinary(rawScan.endKey) + ")");
if (rawScan.fuzzyKeys != null && rawScan.fuzzyKeys.size() != 0) {
- info.append("\nFuzzy key counts: " + rawScan.fuzzyKeys.size());
+ info.append(" Fuzzy key counts: " + rawScan.fuzzyKeys.size());
info.append(". Fuzzy keys : ");
info.append(rawScan.getFuzzyKeyAsString());
} else {
- info.append("\nNo Fuzzy Key");
+ info.append(", No Fuzzy Key");
}
logger.info(info.toString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 3f00566..ee5ae96 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -21,7 +21,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.gridtable.EmptyGTScanner;
@@ -56,7 +55,7 @@ public class CubeSegmentScanner implements IGTScanner {
final Cuboid cuboid;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
- Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) throws NotEnoughGTInfoException {
+ Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) {
this.cuboid = cuboid;
this.cubeSeg = cubeSeg;
this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index ab8c80f..df0bb84 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -11,7 +11,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -96,14 +95,11 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<CubeSegmentScanner> scanners = Lists.newArrayList();
for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
CubeSegmentScanner scanner;
- try {
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation);
- } catch (NotEnoughGTInfoException e) {
- //deal with empty cube segment
- logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata", cubeSeg);
- logger.info("error stack", e);
+ if (cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
continue;
}
+ scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation);
scanners.add(scanner);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
index 361b1dd..c2ffdba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
@@ -61,7 +61,7 @@ public class RawScan {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
- buf.append(System.lineSeparator());
+ buf.append(";");
}
return buf.toString();
}
[07/50] [abbrv] kylin git commit: minor,
UI disable modelname edit when edit model
Posted by sh...@apache.org.
minor, UI disable modelname edit when edit model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ecd12669
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ecd12669
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ecd12669
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: ecd12669a7d7f8a2febbc5199c5291ccfbf339d9
Parents: 2f44970
Author: Jason <ji...@163.com>
Authored: Fri Mar 4 17:58:30 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Mar 4 17:59:12 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/model_info.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecd12669/webapp/app/partials/modelDesigner/model_info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_info.html b/webapp/app/partials/modelDesigner/model_info.html
index a290bfb..bace322 100644
--- a/webapp/app/partials/modelDesigner/model_info.html
+++ b/webapp/app/partials/modelDesigner/model_info.html
@@ -31,7 +31,7 @@
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$sbumitted)}">
<!-- edit -->
- <input ng-if="state.mode=='edit'" name="model_name" type="text" class="form-control"
+ <input ng-if="state.mode=='edit'" name="model_name" type="text" class="form-control" ng-disabled="{{modelMode=='editExistModel'}}"
ng-model="modelsManager.selectedModel.name" required
placeholder="You can use letters, numbers, and '_'"
ng-maxlength=100 ng-pattern="/^\w+$/" />
[37/50] [abbrv] kylin git commit: KYLIN-1226 apply in-mem conf only
to in-mem mr job
Posted by sh...@apache.org.
KYLIN-1226 apply in-mem conf only to in-mem mr job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0232f17e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0232f17e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0232f17e
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 0232f17ed74a3354c7ffd6647d159a8581626861
Parents: 5e13bba
Author: Li Yang <li...@apache.org>
Authored: Fri Mar 11 15:03:40 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Mar 11 15:04:19 2016 +0800
----------------------------------------------------------------------
build/conf/kylin_job_conf.xml | 14 -------------
.../apache/kylin/common/KylinConfigBase.java | 21 +++++++++++++++++++-
.../kylin/engine/mr/steps/InMemCuboidJob.java | 11 ++++++++++
3 files changed, 31 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 052021c..3ecaeda 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -55,20 +55,6 @@
<description>Block replication</description>
</property>
- <!--Properties for calculating cube by splits (in-mem), with which each Mapper need more mem to hold a full cube segment -->
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx2500m</value>
- </property>
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>3072</value>
- </property>
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>200</value>
- </property>
-
<property>
<name>mapred.task.timeout</name>
<value>3600000</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 487f78e..6a47321 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
+import java.util.Map;
import java.util.Properties;
import java.util.SortedSet;
import java.util.regex.Matcher;
@@ -33,6 +34,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@SuppressWarnings("serial")
@@ -476,11 +478,28 @@ public class KylinConfigBase implements Serializable {
}
public int getCubingInMemSamplingPercent() {
- int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inMem.sampling.percent", "30"));
+ int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inmem.sampling.percent", "30"));
percent = Math.max(percent, 1);
percent = Math.min(percent, 100);
return percent;
}
+
+ public Map<String, String> getCubingInMemMRJobConfOverride() {
+ String str = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", //
+ "mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072; mapreduce.task.io.sort.mb=200");
+ Map<String, String> result = Maps.newHashMap();
+ for (String pair : str.split(";")) {
+ int cut = pair.indexOf('=');
+ if (cut < 0)
+ continue;
+ String k = pair.substring(0, cut).trim();
+ String v = pair.substring(cut + 1).trim();
+ if (k.isEmpty() || v.isEmpty())
+ continue;
+ result.put(k, v);
+ }
+ return result;
+ }
public String getHbaseDefaultCompressionCodec() {
return getOptional("kylin.hbase.default.compression.codec", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/0232f17e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index d197252..2319451 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -20,8 +20,10 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -97,6 +99,9 @@ public class InMemCuboidJob extends AbstractHadoopJob {
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
logger.info("Starting: " + job.getJobName());
+
+ // some special tuning for in-mem MR job
+ overrideJobConf(job.getConfiguration(), config);
setJobClasspath(job);
@@ -143,6 +148,12 @@ public class InMemCuboidJob extends AbstractHadoopJob {
}
}
+ private void overrideJobConf(Configuration jobConf, KylinConfig kylinConfig) {
+ for (Entry<String, String> entry : kylinConfig.getCubingInMemMRJobConfOverride().entrySet()) {
+ jobConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+
private int calculateReducerNum(CubeSegment cubeSeg) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
[40/50] [abbrv] kylin git commit: KYLIN-1286 fix java license files
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
index 9609442..ac3ced6 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/monitor/StreamingMonitor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.engine.streaming.monitor;
import java.io.PrintWriter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
index c6ceea9..8747916 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.invertedindex.model;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
index 03b4068..6b21282 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.invertedindex.measure;
import org.apache.kylin.metadata.datatype.DataType;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index a5680d6..cccb604 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.jdbc;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
index 04567cb..1a1deda 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.jdbc;
import java.io.InputStream;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
index 8bf35e2..e537efc 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.jdbc.util;
import java.util.Enumeration;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
----------------------------------------------------------------------
diff --git a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties b/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
index c16246b..cab76a1 100644
--- a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
+++ b/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
driver.name=Kylin JDBC Driver
driver.version=0.1
product.name=Kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/kylin-it/src/test/resources/logging.properties
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/logging.properties b/kylin-it/src/test/resources/logging.properties
index 5d2afe2..8f2713d 100644
--- a/kylin-it/src/test/resources/logging.properties
+++ b/kylin-it/src/test/resources/logging.properties
@@ -1,5 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
handlers=java.util.logging.ConsoleHandler
.level=INFO
-#org.apache.calcite.plan.RelOptPlanner.level=FINE
+#org.apache.calcite.plan.RelOptPlanner.level=FINE
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=org.apache.kylin.common.util.MyLogFormatter
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index f583408..2220113 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.rest.security;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 1f147ef..81a809b 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.rest.security;
import net.sf.ehcache.Cache;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index b98191f..193f5a8 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.rest.security;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index 99737a0..d0aa0ed 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
/**
* This file is licensed to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index f02a361..5a48e83 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.rest.security;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java b/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
index c747e66..78f9d5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.rest.security;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java
index eb1150e..20cc15e 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaClusterConfig.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.source.kafka.config;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
index 1846157..473ff70 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.source.kafka.util;
import com.fasterxml.jackson.databind.ObjectMapper;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
index d496041..f506999 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.source.kafka.util;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a44a57c0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index eb5ac9f..8563a5e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
[19/50] [abbrv] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6
(with Edward Zhang)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
index 7e530db..97b1ef2 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
@@ -44,7 +44,6 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
conf.set("fs.default.name", "file:///");
conf.set("mapreduce.framework.name", "local");
conf.set("mapreduce.application.framework.path", "");
-
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
index 86c8d1d..6852998 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
@@ -118,7 +118,7 @@ public class KylinConnection extends AvaticaConnection {
ArrayList<ColumnMetaData> columns = new ArrayList<ColumnMetaData>();
Map<String, Object> internalParams = Collections.<String, Object> emptyMap();
- return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY);
+ return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY, Meta.StatementType.SELECT);
}
private KylinJdbcFactory factory() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
index f1a4939..6aae983 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
@@ -33,6 +33,7 @@ import org.apache.calcite.avatica.AvaticaStatement;
import org.apache.calcite.avatica.Meta.Frame;
import org.apache.calcite.avatica.Meta.Signature;
import org.apache.calcite.avatica.Meta.StatementHandle;
+import org.apache.calcite.avatica.QueryState;
import org.apache.calcite.avatica.UnregisteredDriver;
/**
@@ -93,9 +94,9 @@ public class KylinJdbcFactory implements AvaticaFactory {
}
@Override
- public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException {
+ public AvaticaResultSet newResultSet(AvaticaStatement statement, QueryState state, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException {
AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature);
- return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, firstFrame);
+ return new KylinResultSet(statement, state, signature, resultSetMetaData, timeZone, firstFrame);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index cae39ad..05f7983 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -30,6 +30,10 @@ import java.util.regex.Pattern;
import org.apache.calcite.avatica.AvaticaUtils;
import org.apache.calcite.avatica.ColumnMetaData;
import org.apache.calcite.avatica.MetaImpl;
+import org.apache.calcite.avatica.MissingResultsException;
+import org.apache.calcite.avatica.NoSuchStatementException;
+import org.apache.calcite.avatica.QueryState;
+import org.apache.calcite.avatica.remote.TypedValue;
import com.google.common.collect.ImmutableList;
@@ -55,6 +59,13 @@ public class KylinMeta extends MetaImpl {
result.signature = connection().mockPreparedSignature(sql);
return result;
}
+
+ // real execution happens in KylinResultSet.execute()
+ @Override
+ public ExecuteResult execute(StatementHandle sh, List<TypedValue> parameterValues, long maxRowCount) throws NoSuchStatementException {
+ final MetaResultSet metaResultSet = MetaResultSet.create(sh.connectionId, sh.id, false, sh.signature, null);
+ return new ExecuteResult(ImmutableList.of(metaResultSet));
+ }
// mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute()
@Override
@@ -91,24 +102,24 @@ public class KylinMeta extends MetaImpl {
}
@Override
- public MetaResultSet getTableTypes() {
+ public MetaResultSet getTableTypes(ConnectionHandle ch) {
return createResultSet(metaTableTypes, MetaTableType.class, "TABLE_TYPE");
}
@Override
- public MetaResultSet getCatalogs() {
+ public MetaResultSet getCatalogs(ConnectionHandle ch) {
List<KMetaCatalog> catalogs = getMetaProject().catalogs;
return createResultSet(catalogs, KMetaCatalog.class, "TABLE_CAT");
}
@Override
- public MetaResultSet getSchemas(String catalog, Pat schemaPattern) {
+ public MetaResultSet getSchemas(ConnectionHandle ch, String catalog, Pat schemaPattern) {
List<KMetaSchema> schemas = getMetaProject().getSchemas(catalog, schemaPattern);
return createResultSet(schemas, KMetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG");
}
@Override
- public MetaResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
+ public MetaResultSet getTables(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
List<KMetaTable> tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList);
return createResultSet(tables, KMetaTable.class, //
"TABLE_CAT", //
@@ -124,7 +135,7 @@ public class KylinMeta extends MetaImpl {
}
@Override
- public MetaResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
+ public MetaResultSet getColumns(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
List<KMetaColumn> columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern);
return createResultSet(columns, KMetaColumn.class, //
"TABLE_CAT", //
@@ -172,7 +183,7 @@ public class KylinMeta extends MetaImpl {
}
CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames);
- Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory);
+ Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory, StatementType.SELECT);
StatementHandle sh = this.createStatement(connection().handle);
Frame frame = new Frame(0, true, iterable);
@@ -355,4 +366,27 @@ public class KylinMeta extends MetaImpl {
}
}
+ @Override
+ public Frame fetch(StatementHandle h, long offset, int fetchMaxRowCount) throws NoSuchStatementException, MissingResultsException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public boolean syncResults(StatementHandle sh, QueryState state, long offset) throws NoSuchStatementException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public void commit(ConnectionHandle ch) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void rollback(ConnectionHandle ch) {
+ // TODO Auto-generated method stub
+
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
index b14865b..04567cb 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
@@ -18,6 +18,8 @@ public class KylinPreparedStatement extends AvaticaPreparedStatement {
protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
+ if (this.handle.signature == null)
+ this.handle.signature = signature;
}
protected List<Object> getParameterValues2() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index b6ac261..2b186bf 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -30,12 +30,13 @@ import org.apache.calcite.avatica.AvaticaStatement;
import org.apache.calcite.avatica.Meta.Frame;
import org.apache.calcite.avatica.Meta.Signature;
import org.apache.calcite.avatica.MetaImpl;
+import org.apache.calcite.avatica.QueryState;
import org.apache.kylin.jdbc.IRemoteClient.QueryResult;
public class KylinResultSet extends AvaticaResultSet {
- public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) {
- super(statement, signature, resultSetMetaData, timeZone, firstFrame);
+ public KylinResultSet(AvaticaStatement statement, QueryState state, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) {
+ super(statement, state, signature, resultSetMetaData, timeZone, firstFrame);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/logging.properties
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/logging.properties b/kylin-it/src/test/resources/logging.properties
index a925478..5d2afe2 100644
--- a/kylin-it/src/test/resources/logging.properties
+++ b/kylin-it/src/test/resources/logging.properties
@@ -1,5 +1,5 @@
handlers=java.util.logging.ConsoleHandler
.level=INFO
-#org.eigenbase.relopt.RelOptPlanner.level=FINEST
+#org.apache.calcite.plan.RelOptPlanner.level=FINE
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=org.apache.kylin.common.util.MyLogFormatter
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query92.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query92.sql b/kylin-it/src/test/resources/query/sql/query92.sql
new file mode 100644
index 0000000..e551a45
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query92.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where meta_categ_name not in ('', 'a')
+ group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query92.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query92.sql.disabled b/kylin-it/src/test/resources/query/sql/query92.sql.disabled
deleted file mode 100644
index e551a45..0000000
--- a/kylin-it/src/test/resources/query/sql/query92.sql.disabled
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select meta_categ_name, count(1) as cnt, sum(price) as GMV
-
- from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
-
- where meta_categ_name not in ('', 'a')
- group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query93.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query93.sql b/kylin-it/src/test/resources/query/sql/query93.sql
new file mode 100644
index 0000000..cc6dca5
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query93.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where meta_categ_name is not null
+ group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query93.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query93.sql.disabled b/kylin-it/src/test/resources/query/sql/query93.sql.disabled
deleted file mode 100644
index cc6dca5..0000000
--- a/kylin-it/src/test/resources/query/sql/query93.sql.disabled
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select meta_categ_name, count(1) as cnt, sum(price) as GMV
-
- from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
-
- where meta_categ_name is not null
- group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query94.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query94.sql b/kylin-it/src/test/resources/query/sql/query94.sql
new file mode 100644
index 0000000..c7899fd
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query94.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where meta_categ_name not in ('Unknown')
+ group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query94.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query94.sql.disabled b/kylin-it/src/test/resources/query/sql/query94.sql.disabled
deleted file mode 100644
index c7899fd..0000000
--- a/kylin-it/src/test/resources/query/sql/query94.sql.disabled
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select meta_categ_name, count(1) as cnt, sum(price) as GMV
-
- from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
-
- where meta_categ_name not in ('Unknown')
- group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query95.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query95.sql b/kylin-it/src/test/resources/query/sql/query95.sql
new file mode 100644
index 0000000..578b93f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query95.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines')
+ group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/kylin-it/src/test/resources/query/sql/query95.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query95.sql.disabled b/kylin-it/src/test/resources/query/sql/query95.sql.disabled
deleted file mode 100644
index 578b93f..0000000
--- a/kylin-it/src/test/resources/query/sql/query95.sql.disabled
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select meta_categ_name, count(1) as cnt, sum(price) as GMV
-
- from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
-
- where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines')
- group by meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2e42841..501bf77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,7 @@
<spring.boot.version>1.2.7.RELEASE</spring.boot.version>
<!-- Calcite Version -->
- <calcite.version>1.4.0-incubating</calcite.version>
+ <calcite.version>1.6.0</calcite.version>
<!-- Curator.version Version -->
<curator.version>2.6.0</curator.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
index 10047e6..c31d1d0 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
@@ -63,7 +63,7 @@ public class OLAPJoinRule extends ConverterRule {
try {
newRel = new OLAPJoinRel(cluster, traitSet, left, right, //
info.getEquiCondition(left, right, cluster.getRexBuilder()), //
- info.leftKeys, info.rightKeys, join.getJoinType(), join.getVariablesStopped());
+ info.leftKeys, info.rightKeys, join.getVariablesSet(), join.getJoinType());
} catch (InvalidRelException e) {
// Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index eed5636..9414757 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -37,6 +37,7 @@ import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -60,6 +61,7 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.query.schema.OLAPTable;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -122,8 +124,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
@@ -280,6 +282,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
fillbackOptimizedColumn();
ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
+ RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
for (int i = 0; i < this.aggregations.size(); i++) {
FunctionDesc aggFunc = this.aggregations.get(i);
@@ -290,7 +293,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
if (aggFunc.needRewrite()) {
String rewriteFieldName = aggFunc.getRewriteFieldName();
- this.context.rewriteFields.put(rewriteFieldName, null);
+ RelDataType rewriteFieldType = OLAPTable.createSqlType(typeFactory, aggFunc.getRewriteFieldType(), true);
+ this.context.rewriteFields.put(rewriteFieldName, rewriteFieldType);
TblColRef column = buildRewriteColumn(aggFunc);
this.context.metricsColumns.add(column);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index a847890..5ea138f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -252,8 +253,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 09a0b17..efe404b 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -43,8 +43,10 @@ import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -73,8 +75,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
public OLAPJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, //
RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, //
- JoinRelType joinType, Set<String> variablesStopped) throws InvalidRelException {
- super(cluster, traits, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped);
+ Set<CorrelationId> variablesSet, JoinRelType joinType) throws InvalidRelException {
+ super(cluster, traits, left, right, condition, leftKeys, rightKeys, variablesSet, joinType);
Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION);
this.rowType = getRowType();
this.isTopJoin = false;
@@ -87,7 +89,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
assert joinInfo.isEqui();
try {
- return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped);
+ return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, variablesSet, joinType);
} catch (InvalidRelException e) {
// Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
@@ -95,13 +97,13 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
- public double getRows() {
- return super.getRows() * 0.1;
+ public double estimateRowCount(RelMetadataQuery mq) {
+ return super.estimateRowCount(mq) * 0.1;
}
@Override
@@ -225,8 +227,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
RexNode.class, //
ImmutableIntList.class, //
ImmutableIntList.class, //
- JoinRelType.class, //
- Set.class);
+ Set.class, //
+ JoinRelType.class);
constr.setAccessible(true);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -238,7 +240,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
if (this.hasSubQuery) {
try {
return constr.newInstance(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
- inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, joinType, variablesStopped);
+ inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, variablesSet, joinType);
} catch (Exception e) {
throw new IllegalStateException("Can't create EnumerableJoin!", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 82aa9de..988d1fa 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -31,6 +31,7 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.SingleRel;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
@@ -59,8 +60,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index b0436e2..8e454c9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -86,8 +87,8 @@ public class OLAPProjectRel extends Project implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
index c3e0595..64a6c3c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
@@ -32,6 +32,7 @@ import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexNode;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -58,8 +59,8 @@ public class OLAPSortRel extends Sort implements OLAPRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 675fd99..bcef413 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -40,14 +40,17 @@ import org.apache.calcite.plan.volcano.AbstractConverter.ExpandConversionRule;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.rules.JoinCommuteRule;
+import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SortJoinTransposeRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -147,6 +150,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
planner.removeRule(AggregateProjectMergeRule.INSTANCE);
planner.removeRule(FilterProjectTransposeRule.INSTANCE);
+ planner.removeRule(SortJoinTransposeRule.INSTANCE);
+ planner.removeRule(JoinPushExpressionsRule.INSTANCE);
// distinct count will be split into a separated query that is joined with the left query
planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
@@ -165,8 +170,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq).multiplyBy(.05);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 7053694..de7e7e2 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.convert.ConverterImpl;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.kylin.metadata.realization.IRealization;
@@ -56,8 +57,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.05);
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ // huge cost to ensure OLAPToEnumerableConverter only appears once in rel tree
+ return planner.getCostFactory().makeCost(1E100, 0, 0);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index a8789ea..12ed6e5 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -43,6 +43,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -128,21 +129,21 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : exposedColumns) {
- RelDataType sqlType = createSqlType(typeFactory, column);
+ RelDataType sqlType = createSqlType(typeFactory, column.getType(), column.isNullable());
sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory);
fieldInfo.add(column.getName(), sqlType);
}
return typeFactory.createStructType(fieldInfo);
}
- private RelDataType createSqlType(RelDataTypeFactory typeFactory, ColumnDesc column) {
- SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(column.getTypeName());
+ public static RelDataType createSqlType(RelDataTypeFactory typeFactory, DataType dataType, boolean isNullable) {
+ SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(dataType.getName());
if (sqlTypeName == null)
- throw new IllegalArgumentException("Unrecognized column type " + column.getTypeName() + " from " + column);
-
- int precision = column.getTypePrecision();
- int scale = column.getTypeScale();
+ throw new IllegalArgumentException("Unrecognized data type " + dataType);
+ int precision = dataType.getPrecision();
+ int scale = dataType.getScale();
+
RelDataType result;
if (precision >= 0 && scale >= 0)
result = typeFactory.createSqlType(sqlTypeName, precision, scale);
@@ -152,7 +153,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
result = typeFactory.createSqlType(sqlTypeName);
// due to left join and uncertain data quality, dimension value can be null
- if (column.isNullable()) {
+ if (isNullable) {
result = typeFactory.createTypeWithNullability(result, true);
} else {
result = typeFactory.createTypeWithNullability(result, false);
[35/50] [abbrv] kylin git commit: clean code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 0a6c123..0b45795 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -22,12 +22,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -38,7 +34,6 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
@@ -52,16 +47,12 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
public class MergeStatisticsStep extends AbstractExecutable {
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String MERGING_SEGMENT_IS = "mergingSegmentIds";
- private static final String MERGED_STATISTICS_PATH = "mergedStatisticsPath";
protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
public MergeStatisticsStep() {
@@ -73,16 +64,16 @@ public class MergeStatisticsStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig kylinConf = context.getConfig();
final CubeManager mgr = CubeManager.getInstance(kylinConf);
- final CubeInstance cube = mgr.getCube(getCubeName());
- final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
+ final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
Configuration conf = HadoopUtil.getCurrentConfiguration();
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
int averageSamplingPercentage = 0;
- for (String segmentId : this.getMergingSegmentIds()) {
- String fileKey = CubeSegment.getStatisticsResourcePath(getCubeName(), segmentId);
+ for (String segmentId : CubingExecutableUtil.getMergingSegmentIds(this.getParams())) {
+ String fileKey = CubeSegment.getStatisticsResourcePath(CubingExecutableUtil.getCubeName(this.getParams()), segmentId);
InputStream is = rs.getResource(fileKey).inputStream;
File tempFile = null;
FileOutputStream tempFileStream = null;
@@ -126,9 +117,9 @@ public class MergeStatisticsStep extends AbstractExecutable {
tempFile.delete();
}
}
- averageSamplingPercentage = averageSamplingPercentage / this.getMergingSegmentIds().size();
- CuboidStatsUtil.writeCuboidStatistics(conf, new Path(getMergedStatisticsPath()), cuboidHLLMap, averageSamplingPercentage);
- Path statisticsFilePath = new Path(getMergedStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ averageSamplingPercentage = averageSamplingPercentage / CubingExecutableUtil.getMergingSegmentIds(this.getParams()).size();
+ CuboidStatsUtil.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage);
+ Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
FileSystem fs = statisticsFilePath.getFileSystem(conf);
FSDataInputStream is = fs.open(statisticsFilePath);
try {
@@ -146,45 +137,4 @@ public class MergeStatisticsStep extends AbstractExecutable {
}
}
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setMergingSegmentIds(List<String> ids) {
- setParam(MERGING_SEGMENT_IS, StringUtils.join(ids, ","));
- }
-
- private List<String> getMergingSegmentIds() {
- final String ids = getParam(MERGING_SEGMENT_IS);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
- public void setMergedStatisticsPath(String path) {
- setParam(MERGED_STATISTICS_PATH, path);
- }
-
- private String getMergedStatisticsPath() {
- return getParam(MERGED_STATISTICS_PATH);
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 1dbce8e..ff9be44 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -131,7 +131,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
if (myChildren == null || myChildren.size() == 0) {
context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L);
skipCounter++;
- if (skipCounter % BatchConstants.COUNTER_MAX == 0) {
+ if (skipCounter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Skipped " + skipCounter + " records!");
}
return;
@@ -140,7 +140,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L);
handleCounter++;
- if (handleCounter % BatchConstants.COUNTER_MAX == 0) {
+ if (handleCounter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
logger.info("Handled " + handleCounter + " records!");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 9314b88..288ca6a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -47,11 +47,6 @@ import org.apache.kylin.metadata.model.MeasureDesc;
*/
public class SaveStatisticsStep extends AbstractExecutable {
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String STATISTICS_PATH = "statisticsPath";
- private static final String CUBING_JOB_ID = "cubingJobId";
-
public SaveStatisticsStep() {
super();
}
@@ -60,15 +55,15 @@ public class SaveStatisticsStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig kylinConf = context.getConfig();
final CubeManager mgr = CubeManager.getInstance(kylinConf);
- final CubeInstance cube = mgr.getCube(getCubeName());
- final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
+ final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
- Path statisticsFilePath = new Path(getStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
if (!fs.exists(statisticsFilePath))
- throw new IOException("File " + statisticsFilePath + " does not exists;");
+ throw new IOException("File " + statisticsFilePath + " does not exists");
FSDataInputStream is = fs.open(statisticsFilePath);
try {
@@ -105,7 +100,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
break;
}
}
-
+
if (memoryHungry == true) {
alg = AlgorithmEnum.LAYER;
} else if ("random".equalsIgnoreCase(algPref)) { // for testing
@@ -120,40 +115,8 @@ public class SaveStatisticsStep extends AbstractExecutable {
}
logger.info("The cube algorithm for " + seg + " is " + alg);
- CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
+ CubingJob cubingJob = (CubingJob) executableManager.getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
cubingJob.setAlgorithm(alg);
}
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setStatisticsPath(String path) {
- this.setParam(STATISTICS_PATH, path);
- }
-
- private String getStatisticsPath() {
- return getParam(STATISTICS_PATH);
- }
-
- public void setCubingJobId(String id) {
- setParam(CUBING_JOB_ID, id);
- }
-
- private String getCubingJobId() {
- return getParam(CUBING_JOB_ID);
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index a10fef4..c41aaf1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -33,50 +33,22 @@ import org.apache.kylin.job.execution.ExecuteResult;
*/
public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
- private static final String SEGMENT_ID = "segmentId";
- private static final String CUBE_NAME = "cubeName";
- private static final String CUBING_JOB_ID = "cubingJobId";
-
public UpdateCubeInfoAfterBuildStep() {
super();
}
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setCubingJobId(String id) {
- setParam(CUBING_JOB_ID, id);
- }
-
- private String getCubingJobId() {
- return getParam(CUBING_JOB_ID);
- }
-
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(getCubeName());
- final CubeSegment segment = cube.getSegmentById(getSegmentId());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
- CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
+ CubingJob cubingJob = (CubingJob) executableManager.getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long sourceCount = cubingJob.findSourceRecordCount();
long sourceSizeBytes = cubingJob.findSourceSizeBytes();
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
- segment.setLastBuildJobID(getCubingJobId());
+ segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
@@ -90,5 +62,5 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index 527572b..d3ed68a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -40,10 +40,6 @@ import com.google.common.collect.Lists;
*/
public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
- private static final String CUBING_JOB_ID = "cubingJobId";
private final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
@@ -53,18 +49,18 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeInstance cube = cubeManager.getCube(getCubeName());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- CubeSegment mergedSegment = cube.getSegmentById(getSegmentId());
+ CubeSegment mergedSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
if (mergedSegment == null) {
- return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
+ return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + CubingExecutableUtil.getSegmentId(this.getParams()));
}
- CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
+ CubingJob cubingJob = (CubingJob) executableManager.getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
// collect source statistics
- List<String> mergingSegmentIds = getMergingSegmentIds();
+ List<String> mergingSegmentIds = CubingExecutableUtil.getMergingSegmentIds(this.getParams());
if (mergingSegmentIds.isEmpty()) {
return new ExecuteResult(ExecuteResult.State.FAILED, "there are no merging segments");
}
@@ -80,7 +76,7 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
mergedSegment.setSizeKB(cubeSizeBytes / 1024);
mergedSegment.setInputRecords(sourceCount);
mergedSegment.setInputRecordsSize(sourceSize);
- mergedSegment.setLastBuildJobID(getCubingJobId());
+ mergedSegment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
mergedSegment.setLastBuildTime(System.currentTimeMillis());
try {
@@ -92,45 +88,4 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
}
}
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setMergingSegmentIds(List<String> ids) {
- setParam(MERGING_SEGMENT_IDS, StringUtils.join(ids, ","));
- }
-
- private List<String> getMergingSegmentIds() {
- final String ids = getParam(MERGING_SEGMENT_IDS);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
- public void setCubingJobId(String id) {
- setParam(CUBING_JOB_ID, id);
- }
-
- private String getCubingJobId() {
- return getParam(CUBING_JOB_ID);
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 285729f..981dac3 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ClassUtil;
@@ -103,7 +104,7 @@ import java.util.concurrent.LinkedBlockingQueue;
public class SparkCubing extends AbstractApplication {
private static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Hive Intermediate Table").create("hiveTable");
- private static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName("cube").hasArg().isRequired(true).withDescription("Cube Name").create("cubeName");
+ private static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube Name").create(BatchConstants.ARG_CUBE_NAME);
private static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName("segment").hasArg().isRequired(true).withDescription("Cube Segment Id").create("segmentId");
private static final Option OPTION_CONF_PATH = OptionBuilder.withArgName("confPath").hasArg().isRequired(true).withDescription("Configuration Path").create("confPath");
private static final Option OPTION_COPROCESSOR = OptionBuilder.withArgName("coprocessor").hasArg().isRequired(true).withDescription("Coprocessor Jar Path").create("coprocessor");
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
index 9f17d60..05246f4 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
@@ -18,12 +18,12 @@
package org.apache.kylin.engine.spark;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.IMROutput2;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.MRUtil;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.slf4j.Logger;
@@ -48,7 +48,7 @@ public class SparkCubingJobBuilder extends JobBuilderSupport {
}
public DefaultChainedExecutable build() {
- final CubingJob result = CubingJob.createBuildJob((CubeSegment)seg, submitter, config);
+ final CubingJob result = CubingJob.createBuildJob((CubeSegment) seg, submitter, config);
final String jobId = result.getId();
inputSide.addStepPhase1_CreateFlatTable(result);
@@ -59,7 +59,7 @@ public class SparkCubingJobBuilder extends JobBuilderSupport {
final SparkExecutable sparkExecutable = new SparkExecutable();
sparkExecutable.setClassName(SparkCubing.class.getName());
sparkExecutable.setParam("hiveTable", tableName);
- sparkExecutable.setParam("cubeName", seg.getRealization().getName());
+ sparkExecutable.setParam(CubingExecutableUtil.CUBE_NAME, seg.getRealization().getName());
sparkExecutable.setParam("segmentId", seg.getUuid());
sparkExecutable.setParam("confPath", confPath);
sparkExecutable.setParam("coprocessor", coprocessor);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 1c3e71a..5fea710 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
@@ -100,40 +101,7 @@ public abstract class BasicService {
}
protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
- List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables()).filter(new Predicate<AbstractExecutable>() {
- @Override
- public boolean apply(AbstractExecutable executable) {
- if (executable instanceof CubingJob) {
- if (cubeName == null) {
- return true;
- }
- return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
- } else {
- return false;
- }
- }
- }).transform(new Function<AbstractExecutable, CubingJob>() {
- @Override
- public CubingJob apply(AbstractExecutable executable) {
- return (CubingJob) executable;
- }
- }).filter(new Predicate<CubingJob>() {
- @Override
- public boolean apply(CubingJob executable) {
- if (null == projectName || null == getProjectManager().getProject(projectName)) {
- return true;
- } else {
- ProjectInstance project = getProjectManager().getProject(projectName);
- return project.containsRealization(RealizationType.CUBE, executable.getCubeName());
- }
- }
- }).filter(new Predicate<CubingJob>() {
- @Override
- public boolean apply(CubingJob executable) {
- return statusList.contains(allOutputs.get(executable.getId()).getState());
- }
- }));
- return results;
+ return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs);
}
protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) {
@@ -144,7 +112,7 @@ public abstract class BasicService {
if (cubeName == null) {
return true;
}
- return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
+ return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName);
} else {
return false;
}
@@ -161,7 +129,7 @@ public abstract class BasicService {
return true;
} else {
ProjectInstance project = getProjectManager().getProject(projectName);
- return project.containsRealization(RealizationType.CUBE, executable.getCubeName());
+ return project.containsRealization(RealizationType.CUBE, CubingExecutableUtil.getCubeName(executable.getParams()));
}
}
}).filter(new Predicate<CubingJob>() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 155593a..cd8eef9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -19,7 +19,13 @@
package org.apache.kylin.rest.service;
import java.io.IOException;
-import java.util.*;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.cube.CubeInstance;
@@ -30,6 +36,7 @@ import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.JobStatusEnum;
@@ -84,7 +91,7 @@ public class JobService extends BasicService {
}
public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
- Calendar calendar= Calendar.getInstance();
+ Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
long currentTimeMillis = calendar.getTimeInMillis();
long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
@@ -153,22 +160,22 @@ public class JobService extends BasicService {
private long getTimeStartInMillis(Calendar calendar, JobTimeFilterEnum timeFilter) {
switch (timeFilter) {
- case LAST_ONE_DAY:
- calendar.add(Calendar.DAY_OF_MONTH, -1);
- return calendar.getTimeInMillis();
- case LAST_ONE_WEEK:
- calendar.add(Calendar.WEEK_OF_MONTH, -1);
- return calendar.getTimeInMillis();
- case LAST_ONE_MONTH:
- calendar.add(Calendar.MONTH, -1);
- return calendar.getTimeInMillis();
- case LAST_ONE_YEAR:
- calendar.add(Calendar.YEAR, -1);
- return calendar.getTimeInMillis();
- case ALL:
- return 0;
- default:
- throw new RuntimeException("illegal timeFilter for job history:" + timeFilter);
+ case LAST_ONE_DAY:
+ calendar.add(Calendar.DAY_OF_MONTH, -1);
+ return calendar.getTimeInMillis();
+ case LAST_ONE_WEEK:
+ calendar.add(Calendar.WEEK_OF_MONTH, -1);
+ return calendar.getTimeInMillis();
+ case LAST_ONE_MONTH:
+ calendar.add(Calendar.MONTH, -1);
+ return calendar.getTimeInMillis();
+ case LAST_ONE_YEAR:
+ calendar.add(Calendar.YEAR, -1);
+ return calendar.getTimeInMillis();
+ case ALL:
+ return 0;
+ default:
+ throw new RuntimeException("illegal timeFilter for job history:" + timeFilter);
}
}
@@ -263,8 +270,8 @@ public class JobService extends BasicService {
CubingJob cubeJob = (CubingJob) job;
final JobInstance result = new JobInstance();
result.setName(job.getName());
- result.setRelatedCube(cubeJob.getCubeName());
- result.setRelatedSegment(cubeJob.getSegmentIds());
+ result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setLastModified(cubeJob.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
@@ -288,8 +295,8 @@ public class JobService extends BasicService {
Output output = outputs.get(job.getId());
final JobInstance result = new JobInstance();
result.setName(job.getName());
- result.setRelatedCube(cubeJob.getCubeName());
- result.setRelatedSegment(cubeJob.getSegmentIds());
+ result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index cbfd8c3..68b6ae4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -60,7 +60,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
bindCurrentConfiguration(conf);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
- String tableName = conf.get(BatchConstants.TABLE_NAME);
+ String tableName = conf.get(BatchConstants.CFG_TABLE_NAME);
tableDesc = MetadataManager.getInstance(config).getTableDesc(tableName);
tableInputFormat = MRUtil.getTableInputFormat(tableDesc);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index 70286ab..9162208 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -36,7 +36,6 @@ import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
/**
@@ -79,7 +78,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
setJobClasspath(job);
String table = getOptionValue(OPTION_TABLE);
- job.getConfiguration().set(BatchConstants.TABLE_NAME, table);
+ job.getConfiguration().set(BatchConstants.CFG_TABLE_NAME, table);
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
FileOutputFormat.setOutputPath(job, output);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index c2e2e64..6d77240 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -154,30 +154,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException {
- byte[] startRow = Bytes.toBytes(rangeStart);
- byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
-
- Scan scan = new Scan(startRow, endRow);
- scan.addColumn(B_FAMILY, B_COLUMN_TS);
- scan.addColumn(B_FAMILY, B_COLUMN);
- tuneScanParameters(scan);
-
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- List<RawResource> result = Lists.newArrayList();
- try {
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
- }
- } catch (IOException e) {
- for (RawResource rawResource : result) {
- IOUtils.closeQuietly(rawResource.inputStream);
- }
- throw e;
- } finally {
- IOUtils.closeQuietly(table);
- }
- return result;
+ return getAllResources(rangeStart, rangeEnd, -1L, -1L);
}
@Override
@@ -188,7 +165,10 @@ public class HBaseResourceStore extends ResourceStore {
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
- scan.setFilter(generateTimeFilterList(timeStartInMillis, timeEndInMillis));
+ FilterList filterList = generateTimeFilterList(timeStartInMillis, timeEndInMillis);
+ if (filterList != null) {
+ scan.setFilter(filterList);
+ }
tuneScanParameters(scan);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
@@ -218,11 +198,15 @@ public class HBaseResourceStore extends ResourceStore {
private FilterList generateTimeFilterList(long timeStartInMillis, long timeEndInMillis) {
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
- SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.GREATER, Bytes.toBytes(timeStartInMillis));
- filterList.addFilter(timeStartFilter);
- SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.LESS_OR_EQUAL, Bytes.toBytes(timeEndInMillis));
- filterList.addFilter(timeEndFilter);
- return filterList;
+ if (timeStartInMillis != -1L) {
+ SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.GREATER, Bytes.toBytes(timeStartInMillis));
+ filterList.addFilter(timeStartFilter);
+ }
+ if (timeEndInMillis != -1L) {
+ SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.LESS_OR_EQUAL, Bytes.toBytes(timeEndInMillis));
+ filterList.addFilter(timeEndFilter);
+ }
+ return filterList.getFilters().size() == 0 ? null : filterList;
}
private InputStream getInputStream(String resPath, Result r) throws IOException {
@@ -325,7 +309,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
-
+
if (!fetchContent && !fetchTimestamp) {
get.setCheckExistenceOnly(true);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index a828728..f71d0f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -7,15 +7,16 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.JobBuilderSupport;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.storage.hbase.ii.IIBulkLoadJob;
-import org.apache.kylin.storage.hbase.ii.IICreateHFileJob;
-import org.apache.kylin.storage.hbase.ii.IICreateHTableJob;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.realization.IRealizationSegment;
import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.ii.IIBulkLoadJob;
+import org.apache.kylin.storage.hbase.ii.IICreateHFileJob;
+import org.apache.kylin.storage.hbase.ii.IICreateHTableJob;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -41,16 +42,16 @@ public class HBaseMRSteps extends JobBuilderSupport {
public MapReduceExecutable createRangeRowkeyDistributionStep(String cuboidRootPath, String jobId) {
String inputPath = cuboidRootPath + (cuboidRootPath.endsWith("/") ? "" : "/") + "*";
-
+
MapReduceExecutable rowkeyDistributionStep = new MapReduceExecutable();
rowkeyDistributionStep.setName(ExecutableConstants.STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION);
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg.getRealization().getDataModelDesc());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(jobId));
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "jobname", "Kylin_Region_Splits_Calculator_" + seg.getRealization().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getRowkeyDistributionOutputPath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Region_Splits_Calculator_" + seg.getRealization().getName() + "_Step");
rowkeyDistributionStep.setMapReduceParams(cmd.toString());
rowkeyDistributionStep.setMapReduceJobClass(RangeKeyDistributionJob.class);
@@ -69,10 +70,10 @@ public class HBaseMRSteps extends JobBuilderSupport {
HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "partitions", getRowkeyDistributionOutputPath(jobId) + "/part-r-00000");
- appendExecCmdParameters(cmd, "statisticsenabled", String.valueOf(withStats));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_PARTITION, getRowkeyDistributionOutputPath(jobId) + "/part-r-00000");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_ENABLED, String.valueOf(withStats));
createHtableStep.setJobParams(cmd.toString());
createHtableStep.setJobClass(CreateHTableJob.class);
@@ -82,18 +83,18 @@ public class HBaseMRSteps extends JobBuilderSupport {
public MapReduceExecutable createConvertCuboidToHfileStep(String cuboidRootPath, String jobId) {
String inputPath = cuboidRootPath + (cuboidRootPath.endsWith("/") ? "" : "/") + "*";
-
+
MapReduceExecutable createHFilesStep = new MapReduceExecutable();
createHFilesStep.setName(ExecutableConstants.STEP_NAME_CONVERT_CUBOID_TO_HFILE);
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg.getRealization().getDataModelDesc());
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "partitions", getRowkeyDistributionOutputPath(jobId) + "/part-r-00000_hfile");
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getHFilePath(jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "jobname", "Kylin_HFile_Generator_" + seg.getRealization().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_PARTITION, getRowkeyDistributionOutputPath(jobId) + "/part-r-00000_hfile");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getHFilePath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_HTABLE_NAME, seg.getStorageLocationIdentifier());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_HFile_Generator_" + seg.getRealization().getName() + "_Step");
createHFilesStep.setMapReduceParams(cmd.toString());
createHFilesStep.setMapReduceJobClass(CubeHFileJob.class);
@@ -107,9 +108,9 @@ public class HBaseMRSteps extends JobBuilderSupport {
bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE);
StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "input", getHFilePath(jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getHFilePath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_HTABLE_NAME, seg.getStorageLocationIdentifier());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
bulkLoadStep.setJobParams(cmd.toString());
bulkLoadStep.setJobClass(BulkLoadJob.class);
@@ -125,7 +126,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
}
public List<String> getMergingHTables() {
- final List<CubeSegment> mergingSegments = ((CubeInstance)seg.getRealization()).getMergingSegments((CubeSegment)seg);
+ final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
final List<String> mergingHTables = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
@@ -135,7 +136,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
}
public List<String> getMergingHDFSPaths() {
- final List<CubeSegment> mergingSegments = ((CubeInstance)seg.getRealization()).getMergingSegments((CubeSegment)seg);
+ final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
final List<String> mergingHDFSPaths = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
@@ -187,15 +188,14 @@ public class HBaseMRSteps extends JobBuilderSupport {
jobFlow.addTask(createCreateIIHTableStep(seg));
final String iiPath = rootPath + "*";
-
+
// generate hfiles step
jobFlow.addTask(createConvertIIToHfileStep(seg, iiPath, jobFlow.getId()));
// bulk load step
jobFlow.addTask(createIIBulkLoadStep(seg, jobFlow.getId()));
-
- }
+ }
public void addInvertedIndexGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
String jobId = jobFlow.getId();
@@ -211,14 +211,12 @@ public class HBaseMRSteps extends JobBuilderSupport {
jobFlow.addTask(step);
}
-
-
private HadoopShellExecutable createCreateIIHTableStep(IRealizationSegment seg) {
HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "iiname", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_HTABLE_NAME, seg.getStorageLocationIdentifier());
createHtableStep.setJobParams(cmd.toString());
createHtableStep.setJobClass(IICreateHTableJob.class);
@@ -232,11 +230,11 @@ public class HBaseMRSteps extends JobBuilderSupport {
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg.getRealization().getDataModelDesc());
- appendExecCmdParameters(cmd, "iiname", seg.getRealization().getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getHFilePath(jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "jobname", "Kylin_HFile_Generator_" + seg.getRealization().getName() + "_Step");
+ appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getHFilePath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_HTABLE_NAME, seg.getStorageLocationIdentifier());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_HFile_Generator_" + seg.getRealization().getName() + "_Step");
createHFilesStep.setMapReduceParams(cmd.toString());
createHFilesStep.setMapReduceJobClass(IICreateHFileJob.class);
@@ -249,9 +247,9 @@ public class HBaseMRSteps extends JobBuilderSupport {
bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE);
StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "input", getHFilePath(jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "iiname", seg.getRealization().getName());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getHFilePath(jobId));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_HTABLE_NAME, seg.getStorageLocationIdentifier());
+ appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName());
bulkLoadStep.setJobParams(cmd.toString());
bulkLoadStep.setJobClass(IIBulkLoadJob.class);
@@ -259,5 +257,5 @@ public class HBaseMRSteps extends JobBuilderSupport {
return bulkLoadStep;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 4cc4794..fa62a62 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -76,7 +76,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
CuboidStatsUtil.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
FSDataInputStream inputStream = null;
try {
- inputStream = FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION));
+ inputStream = FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME));
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), inputStream, System.currentTimeMillis());
} finally {
IOUtils.closeQuietly(inputStream);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
index 2ff7356..8b5daa3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
@@ -99,11 +99,11 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
int regionSplitSize = KylinConfig.getInstanceFromEnv().getHBaseRegionCut(cubeCapacity.toString());
int maxRegionCount = KylinConfig.getInstanceFromEnv().getHBaseRegionCountMax();
int minRegionCount = KylinConfig.getInstanceFromEnv().getHBaseRegionCountMin();
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
- job.getConfiguration().set(BatchConstants.HFILE_SIZE_GB, String.valueOf(hfileSizeGB));
- job.getConfiguration().set(BatchConstants.REGION_SPLIT_SIZE, String.valueOf(regionSplitSize));
- job.getConfiguration().set(BatchConstants.REGION_NUMBER_MAX, String.valueOf(maxRegionCount));
- job.getConfiguration().set(BatchConstants.REGION_NUMBER_MIN, String.valueOf(minRegionCount));
+ job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
+ job.getConfiguration().set(BatchConstants.CFG_HFILE_SIZE_GB, String.valueOf(hfileSizeGB));
+ job.getConfiguration().set(BatchConstants.CFG_REGION_SPLIT_SIZE, String.valueOf(regionSplitSize));
+ job.getConfiguration().set(BatchConstants.CFG_REGION_NUMBER_MAX, String.valueOf(maxRegionCount));
+ job.getConfiguration().set(BatchConstants.CFG_REGION_NUMBER_MIN, String.valueOf(minRegionCount));
// The partition file for hfile is sequenece file consists of ImmutableBytesWritable and NullWritable
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), ImmutableBytesWritable.class, NullWritable.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e13bba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
index acdab62..4e53ca4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
@@ -57,24 +57,24 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
- if (context.getConfiguration().get(BatchConstants.OUTPUT_PATH) != null) {
- output = context.getConfiguration().get(BatchConstants.OUTPUT_PATH);
+ if (context.getConfiguration().get(BatchConstants.CFG_OUTPUT_PATH) != null) {
+ output = context.getConfiguration().get(BatchConstants.CFG_OUTPUT_PATH);
}
- if (context.getConfiguration().get(BatchConstants.HFILE_SIZE_GB) != null) {
- hfileSizeGB = Integer.valueOf(context.getConfiguration().get(BatchConstants.HFILE_SIZE_GB));
+ if (context.getConfiguration().get(BatchConstants.CFG_HFILE_SIZE_GB) != null) {
+ hfileSizeGB = Integer.valueOf(context.getConfiguration().get(BatchConstants.CFG_HFILE_SIZE_GB));
}
- if (context.getConfiguration().get(BatchConstants.REGION_SPLIT_SIZE) != null) {
- cut = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_SPLIT_SIZE));
+ if (context.getConfiguration().get(BatchConstants.CFG_REGION_SPLIT_SIZE) != null) {
+ cut = Integer.valueOf(context.getConfiguration().get(BatchConstants.CFG_REGION_SPLIT_SIZE));
}
- if (context.getConfiguration().get(BatchConstants.REGION_NUMBER_MIN) != null) {
- minRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_NUMBER_MIN));
+ if (context.getConfiguration().get(BatchConstants.CFG_REGION_NUMBER_MIN) != null) {
+ minRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.CFG_REGION_NUMBER_MIN));
}
- if (context.getConfiguration().get(BatchConstants.REGION_NUMBER_MAX) != null) {
- maxRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_NUMBER_MAX));
+ if (context.getConfiguration().get(BatchConstants.CFG_REGION_NUMBER_MAX) != null) {
+ maxRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.CFG_REGION_NUMBER_MAX));
}
logger.info("Chosen cut for htable is " + cut + ", max region count=" + maxRegionCount
[21/50] [abbrv] kylin git commit: KYLIN-1343 Upgrade to calcite 1.6
(with Edward Zhang)
Posted by sh...@apache.org.
KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f73abf6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f73abf6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f73abf6c
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: f73abf6c2df97c1c89e7e6edf3865d82350f8472
Parents: 9021f17
Author: Yang Li <li...@apache.org>
Authored: Sun Mar 6 00:06:05 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Mar 8 15:54:05 2016 +0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableJoin.java | 145 -
.../apache/calcite/runtime/SqlFunctions.java | 1317 ---
.../apache/calcite/sql2rel/RelFieldTrimmer.java | 44 +
.../calcite/sql2rel/SqlToRelConverter.java | 8150 ++++++++++--------
.../engine/mr/steps/MergeCuboidJobTest.java | 1 -
.../org/apache/kylin/jdbc/KylinConnection.java | 2 +-
.../org/apache/kylin/jdbc/KylinJdbcFactory.java | 5 +-
.../java/org/apache/kylin/jdbc/KylinMeta.java | 46 +-
.../kylin/jdbc/KylinPreparedStatement.java | 2 +
.../org/apache/kylin/jdbc/KylinResultSet.java | 5 +-
kylin-it/src/test/resources/logging.properties | 2 +-
.../src/test/resources/query/sql/query92.sql | 30 +
.../resources/query/sql/query92.sql.disabled | 30 -
.../src/test/resources/query/sql/query93.sql | 30 +
.../resources/query/sql/query93.sql.disabled | 30 -
.../src/test/resources/query/sql/query94.sql | 30 +
.../resources/query/sql/query94.sql.disabled | 30 -
.../src/test/resources/query/sql/query95.sql | 30 +
.../resources/query/sql/query95.sql.disabled | 30 -
pom.xml | 2 +-
.../kylin/query/optrule/OLAPJoinRule.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 10 +-
.../kylin/query/relnode/OLAPFilterRel.java | 5 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 22 +-
.../kylin/query/relnode/OLAPLimitRel.java | 5 +-
.../kylin/query/relnode/OLAPProjectRel.java | 5 +-
.../apache/kylin/query/relnode/OLAPSortRel.java | 5 +-
.../kylin/query/relnode/OLAPTableScan.java | 9 +-
.../relnode/OLAPToEnumerableConverter.java | 6 +-
.../apache/kylin/query/schema/OLAPTable.java | 17 +-
30 files changed, 4836 insertions(+), 5211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
deleted file mode 100644
index a3c04f8..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * OVERRIDE POINT:
- * - constructor was private instead of protected
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.enumerable;
-
-import java.util.Set;
-
-import org.apache.calcite.linq4j.tree.BlockBuilder;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.InvalidRelException;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelNodes;
-import org.apache.calcite.rel.core.EquiJoin;
-import org.apache.calcite.rel.core.JoinInfo;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.util.BuiltInMethod;
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
-
-import com.google.common.collect.ImmutableList;
-
-/** Implementation of {@link org.apache.calcite.rel.core.Join} in
- * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */
-public class EnumerableJoin extends EquiJoin implements EnumerableRel {
- /** Creates an EnumerableJoin.
- *
- * <p>Use {@link #create} unless you know what you're doing. */
- public EnumerableJoin(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, JoinRelType joinType, Set<String> variablesStopped) throws InvalidRelException {
- super(cluster, traits, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped);
- }
-
- /** Creates an EnumerableJoin. */
- public static EnumerableJoin create(RelNode left, RelNode right, RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, JoinRelType joinType, Set<String> variablesStopped) throws InvalidRelException {
- final RelOptCluster cluster = left.getCluster();
- final RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE);
- return new EnumerableJoin(cluster, traitSet, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped);
- }
-
- @Override
- public EnumerableJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) {
- final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
- assert joinInfo.isEqui();
- try {
- return new EnumerableJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped);
- } catch (InvalidRelException e) {
- // Semantic error not possible. Must be a bug. Convert to
- // internal error.
- throw new AssertionError(e);
- }
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- double rowCount = RelMetadataQuery.getRowCount(this);
-
- // Joins can be flipped, and for many algorithms, both versions are viable
- // and have the same cost. To make the results stable between versions of
- // the planner, make one of the versions slightly more expensive.
- switch (joinType) {
- case RIGHT:
- rowCount = addEpsilon(rowCount);
- break;
- default:
- if (RelNodes.COMPARATOR.compare(left, right) > 0) {
- rowCount = addEpsilon(rowCount);
- }
- }
-
- // Cheaper if the smaller number of rows is coming from the LHS.
- // Model this by adding L log L to the cost.
- final double rightRowCount = right.getRows();
- final double leftRowCount = left.getRows();
- if (Double.isInfinite(leftRowCount)) {
- rowCount = leftRowCount;
- } else {
- rowCount += Util.nLogN(leftRowCount);
- }
- if (Double.isInfinite(rightRowCount)) {
- rowCount = rightRowCount;
- } else {
- rowCount += rightRowCount;
- }
- return planner.getCostFactory().makeCost(rowCount, 0, 0);
- }
-
- private double addEpsilon(double d) {
- assert d >= 0d;
- final double d0 = d;
- if (d < 10) {
- // For small d, adding 1 would change the value significantly.
- d *= 1.001d;
- if (d != d0) {
- return d;
- }
- }
- // For medium d, add 1. Keeps integral values integral.
- ++d;
- if (d != d0) {
- return d;
- }
- // For large d, adding 1 might not change the value. Add .1%.
- // If d is NaN, this still will probably not change the value. That's OK.
- d *= 1.001d;
- return d;
- }
-
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- BlockBuilder builder = new BlockBuilder();
- final Result leftResult = implementor.visitChild(this, 0, (EnumerableRel) left, pref);
- Expression leftExpression = builder.append("left", leftResult.block);
- final Result rightResult = implementor.visitChild(this, 1, (EnumerableRel) right, pref);
- Expression rightExpression = builder.append("right", rightResult.block);
- final PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
- final PhysType keyPhysType = leftResult.physType.project(leftKeys, JavaRowFormat.LIST);
- return implementor.result(physType, builder.append(Expressions.call(leftExpression, BuiltInMethod.JOIN.method, Expressions.list(rightExpression, leftResult.physType.generateAccessor(leftKeys), rightResult.physType.generateAccessor(rightKeys), EnumUtils.joinSelector(joinType, physType, ImmutableList.of(leftResult.physType, rightResult.physType))).append(Util.first(keyPhysType.comparer(), Expressions.constant(null))).append(Expressions.constant(joinType.generatesNullsOnLeft())).append(Expressions.constant(joinType.generatesNullsOnRight())))).toBlock());
- }
-
-}
-
-// End EnumerableJoin.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
deleted file mode 100644
index 236601f..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ /dev/null
@@ -1,1317 +0,0 @@
-/*
- * OVERRIDE POINT:
- * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);`
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.runtime;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.MathContext;
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.text.DecimalFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Pattern;
-
-import org.apache.calcite.DataContext;
-import org.apache.calcite.avatica.util.ByteString;
-import org.apache.calcite.avatica.util.DateTimeUtils;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Linq4j;
-import org.apache.calcite.linq4j.function.Deterministic;
-import org.apache.calcite.linq4j.function.Function1;
-import org.apache.calcite.linq4j.function.NonDeterministic;
-import org.apache.calcite.linq4j.tree.Primitive;
-
-/**
- * Helper methods to implement SQL functions in generated code.
- *
- * <p>Not present: and, or, not (builtin operators are better, because they
- * use lazy evaluation. Implementations do not check for null values; the
- * calling code must do that.</p>
- *
- * <p>Many of the functions do not check for null values. This is intentional.
- * If null arguments are possible, the code-generation framework checks for
- * nulls before calling the functions.</p>
- */
-@SuppressWarnings("UnnecessaryUnboxing")
-@Deterministic
-public class SqlFunctions {
- private static final DecimalFormat DOUBLE_FORMAT = new DecimalFormat("0.0E0");
-
- private static final TimeZone LOCAL_TZ = TimeZone.getDefault();
-
- private static final Function1<List<Object>, Enumerable<Object>> LIST_AS_ENUMERABLE = new Function1<List<Object>, Enumerable<Object>>() {
- public Enumerable<Object> apply(List<Object> list) {
- return Linq4j.asEnumerable(list);
- }
- };
-
- /** Holds, for each thread, a map from sequence name to sequence current
- * value.
- *
- * <p>This is a straw man of an implementation whose main goal is to prove
- * that sequences can be parsed, validated and planned. A real application
- * will want persistent values for sequences, shared among threads. */
- private static final ThreadLocal<Map<String, AtomicLong>> THREAD_SEQUENCES = new ThreadLocal<Map<String, AtomicLong>>() {
- @Override
- protected Map<String, AtomicLong> initialValue() {
- return new HashMap<String, AtomicLong>();
- }
- };
-
- private SqlFunctions() {
- }
-
- /** SQL SUBSTRING(string FROM ... FOR ...) function. */
- public static String substring(String s, int from, int for_) {
- return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
- }
-
- /** SQL SUBSTRING(string FROM ...) function. */
- public static String substring(String s, int from) {
- return s.substring(from - 1);
- }
-
- /** SQL UPPER(string) function. */
- public static String upper(String s) {
- return s.toUpperCase();
- }
-
- /** SQL LOWER(string) function. */
- public static String lower(String s) {
- return s.toLowerCase();
- }
-
- /** SQL INITCAP(string) function. */
- public static String initcap(String s) {
- // Assumes Alpha as [A-Za-z0-9]
- // white space is treated as everything else.
- final int len = s.length();
- boolean start = true;
- final StringBuilder newS = new StringBuilder();
-
- for (int i = 0; i < len; i++) {
- char curCh = s.charAt(i);
- final int c = (int) curCh;
- if (start) { // curCh is whitespace or first character of word.
- if (c > 47 && c < 58) { // 0-9
- start = false;
- } else if (c > 64 && c < 91) { // A-Z
- start = false;
- } else if (c > 96 && c < 123) { // a-z
- start = false;
- curCh = (char) (c - 32); // Uppercase this character
- }
- // else {} whitespace
- } else { // Inside of a word or white space after end of word.
- if (c > 47 && c < 58) { // 0-9
- // noop
- } else if (c > 64 && c < 91) { // A-Z
- curCh = (char) (c + 32); // Lowercase this character
- } else if (c > 96 && c < 123) { // a-z
- // noop
- } else { // whitespace
- start = true;
- }
- }
- newS.append(curCh);
- } // for each character in s
- return newS.toString();
- }
-
- /** SQL CHARACTER_LENGTH(string) function. */
- public static int charLength(String s) {
- return s.length();
- }
-
- /** SQL {@code string || string} operator. */
- public static String concat(String s0, String s1) {
- return s0 + s1;
- }
-
- /** SQL {@code binary || binary} operator. */
- public static ByteString concat(ByteString s0, ByteString s1) {
- return s0.concat(s1);
- }
-
- /** SQL {@code RTRIM} function applied to string. */
- public static String rtrim(String s) {
- return trim_(s, false, true, ' ');
- }
-
- /** SQL {@code LTRIM} function. */
- public static String ltrim(String s) {
- return trim_(s, true, false, ' ');
- }
-
- /** SQL {@code TRIM(... seek FROM s)} function. */
- public static String trim(boolean leading, boolean trailing, String seek, String s) {
- return trim_(s, leading, trailing, seek.charAt(0));
- }
-
- /** SQL {@code TRIM} function. */
- private static String trim_(String s, boolean left, boolean right, char c) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return "";
- }
- if (s.charAt(j - 1) != c) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return "";
- }
- if (s.charAt(i) != c) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- public static ByteString trim(ByteString s) {
- return trim_(s, true, true);
- }
-
- /** Helper for CAST. */
- public static ByteString rtrim(ByteString s) {
- return trim_(s, false, true);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- private static ByteString trim_(ByteString s, boolean left, boolean right) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(j - 1) != 0) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(i) != 0) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1) + r + s.substring(start - 1 + r.length());
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start, int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1) + r + s.substring(start - 1 + length);
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1).concat(r).concat(s.substring(start - 1 + r.length()));
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start, int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1).concat(r).concat(s.substring(start - 1 + length));
- }
-
- /** SQL {@code LIKE} function. */
- public static boolean like(String s, String pattern) {
- final String regex = Like.sqlToRegexLike(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code LIKE} function with escape. */
- public static boolean like(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexLike(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function. */
- public static boolean similar(String s, String pattern) {
- final String regex = Like.sqlToRegexSimilar(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function with escape. */
- public static boolean similar(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexSimilar(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- // =
-
- /** SQL = operator applied to Object values (including String; neither
- * side may be null). */
- public static boolean eq(Object b0, Object b1) {
- return b0.equals(b1);
- }
-
- /** SQL = operator applied to BigDecimal values (neither may be null). */
- public static boolean eq(BigDecimal b0, BigDecimal b1) {
- return b0.stripTrailingZeros().equals(b1.stripTrailingZeros());
- }
-
- // <>
-
- /** SQL <> operator applied to Object values (including String;
- * neither side may be null). */
- public static boolean ne(Object b0, Object b1) {
- return !b0.equals(b1);
- }
-
- /** SQL <> operator applied to BigDecimal values. */
- public static boolean ne(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) != 0;
- }
-
- // <
-
- /** SQL < operator applied to boolean values. */
- public static boolean lt(boolean b0, boolean b1) {
- return compare(b0, b1) < 0;
- }
-
- /** SQL < operator applied to String values. */
- public static boolean lt(String b0, String b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to ByteString values. */
- public static boolean lt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to BigDecimal values. */
- public static boolean lt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) < 0;
- }
-
- // <=
-
- /** SQL ≤ operator applied to boolean values. */
- public static boolean le(boolean b0, boolean b1) {
- return compare(b0, b1) <= 0;
- }
-
- /** SQL ≤ operator applied to String values. */
- public static boolean le(String b0, String b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to ByteString values. */
- public static boolean le(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to BigDecimal values. */
- public static boolean le(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- // >
-
- /** SQL > operator applied to boolean values. */
- public static boolean gt(boolean b0, boolean b1) {
- return compare(b0, b1) > 0;
- }
-
- /** SQL > operator applied to String values. */
- public static boolean gt(String b0, String b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to ByteString values. */
- public static boolean gt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to BigDecimal values. */
- public static boolean gt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) > 0;
- }
-
- // >=
-
- /** SQL ≥ operator applied to boolean values. */
- public static boolean ge(boolean b0, boolean b1) {
- return compare(b0, b1) >= 0;
- }
-
- /** SQL ≥ operator applied to String values. */
- public static boolean ge(String b0, String b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to ByteString values. */
- public static boolean ge(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to BigDecimal values. */
- public static boolean ge(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- // +
-
- /** SQL <code>+</code> operator applied to int values. */
- public static int plus(int b0, int b1) {
- return b0 + b1;
- }
-
- /** SQL <code>+</code> operator applied to int values; left side may be
- * null. */
- public static Integer plus(Integer b0, int b1) {
- return b0 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to int values; right side may be
- * null. */
- public static Integer plus(int b0, Integer b1) {
- return b1 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable int values. */
- public static Integer plus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable long and int values. */
- public static Long plus(Long b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to nullable int and long values. */
- public static Long plus(Integer b0, Long b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to BigDecimal values. */
- public static BigDecimal plus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.add(b1);
- }
-
- // -
-
- /** SQL <code>-</code> operator applied to int values. */
- public static int minus(int b0, int b1) {
- return b0 - b1;
- }
-
- /** SQL <code>-</code> operator applied to int values; left side may be
- * null. */
- public static Integer minus(Integer b0, int b1) {
- return b0 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to int values; right side may be
- * null. */
- public static Integer minus(int b0, Integer b1) {
- return b1 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable int values. */
- public static Integer minus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable long and int values. */
- public static Long minus(Long b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to nullable int and long values. */
- public static Long minus(Integer b0, Long b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to BigDecimal values. */
- public static BigDecimal minus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.subtract(b1);
- }
-
- // /
-
- /** SQL <code>/</code> operator applied to int values. */
- public static int divide(int b0, int b1) {
- return b0 / b1;
- }
-
- /** SQL <code>/</code> operator applied to int values; left side may be
- * null. */
- public static Integer divide(Integer b0, int b1) {
- return b0 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to int values; right side may be
- * null. */
- public static Integer divide(int b0, Integer b1) {
- return b1 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable int values. */
- public static Integer divide(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable long and int values. */
- public static Long divide(Long b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to nullable int and long values. */
- public static Long divide(Integer b0, Long b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to BigDecimal values. */
- public static BigDecimal divide(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64);
- }
-
- // *
-
- /** SQL <code>*</code> operator applied to int values. */
- public static int multiply(int b0, int b1) {
- return b0 * b1;
- }
-
- /** SQL <code>*</code> operator applied to int values; left side may be
- * null. */
- public static Integer multiply(Integer b0, int b1) {
- return b0 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to int values; right side may be
- * null. */
- public static Integer multiply(int b0, Integer b1) {
- return b1 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable int values. */
- public static Integer multiply(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable long and int values. */
- public static Long multiply(Long b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to nullable int and long values. */
- public static Long multiply(Integer b0, Long b1) {
- return (b0 == null || b1 == null) ? null : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to BigDecimal values. */
- public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.multiply(b1);
- }
-
- // EXP
-
- /** SQL <code>EXP</code> operator applied to double values. */
- public static double exp(double b0) {
- return Math.exp(b0);
- }
-
- public static double exp(long b0) {
- return Math.exp(b0);
- }
-
- // POWER
-
- /** SQL <code>POWER</code> operator applied to double values. */
- public static double power(double b0, double b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, long b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, BigDecimal b1) {
- return Math.pow(b0, b1.doubleValue());
- }
-
- // LN
-
- /** SQL {@code LN(number)} function applied to double values. */
- public static double ln(double d) {
- return Math.log(d);
- }
-
- /** SQL {@code LN(number)} function applied to long values. */
- public static double ln(long b0) {
- return Math.log(b0);
- }
-
- /** SQL {@code LN(number)} function applied to BigDecimal values. */
- public static double ln(BigDecimal d) {
- return Math.log(d.doubleValue());
- }
-
- // LOG10
-
- /** SQL <code>LOG10(numeric)</code> operator applied to double values. */
- public static double log10(double b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to long values. */
- public static double log10(long b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to BigDecimal values. */
- public static double log10(BigDecimal d) {
- return Math.log10(d.doubleValue());
- }
-
- // MOD
-
- /** SQL <code>MOD</code> operator applied to byte values. */
- public static byte mod(byte b0, byte b1) {
- return (byte) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to short values. */
- public static short mod(short b0, short b1) {
- return (short) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to int values. */
- public static int mod(int b0, int b1) {
- return b0 % b1;
- }
-
- /** SQL <code>MOD</code> operator applied to long values. */
- public static long mod(long b0, long b1) {
- return b0 % b1;
- }
-
- // temporary
- public static BigDecimal mod(BigDecimal b0, int b1) {
- return mod(b0, BigDecimal.valueOf(b1));
- }
-
- // temporary
- public static int mod(int b0, BigDecimal b1) {
- return mod(b0, b1.intValue());
- }
-
- public static BigDecimal mod(BigDecimal b0, BigDecimal b1) {
- final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
- return bigDecimals[1];
- }
-
- // FLOOR
-
- public static double floor(double b0) {
- return Math.floor(b0);
- }
-
- public static float floor(float b0) {
- return (float) Math.floor(b0);
- }
-
- public static BigDecimal floor(BigDecimal b0) {
- return b0.setScale(0, BigDecimal.ROUND_FLOOR);
- }
-
- /** SQL <code>FLOOR</code> operator applied to byte values. */
- public static byte floor(byte b0, byte b1) {
- return (byte) floor((int) b0, (int) b1);
- }
-
- /** SQL <code>FLOOR</code> operator applied to short values. */
- public static short floor(short b0, short b1) {
- return (short) floor((int) b0, (int) b1);
- }
-
- /** SQL <code>FLOOR</code> operator applied to int values. */
- public static int floor(int b0, int b1) {
- int r = b0 % b1;
- if (r < 0) {
- r += b1;
- }
- return b0 - r;
- }
-
- /** SQL <code>FLOOR</code> operator applied to long values. */
- public static long floor(long b0, long b1) {
- long r = b0 % b1;
- if (r < 0) {
- r += b1;
- }
- return b0 - r;
- }
-
- // temporary
- public static BigDecimal floor(BigDecimal b0, int b1) {
- return floor(b0, BigDecimal.valueOf(b1));
- }
-
- // temporary
- public static int floor(int b0, BigDecimal b1) {
- return floor(b0, b1.intValue());
- }
-
- public static BigDecimal floor(BigDecimal b0, BigDecimal b1) {
- final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
- BigDecimal r = bigDecimals[1];
- if (r.signum() < 0) {
- r = r.add(b1);
- }
- return b0.subtract(r);
- }
-
- // CEIL
-
- public static double ceil(double b0) {
- return Math.ceil(b0);
- }
-
- public static float ceil(float b0) {
- return (float) Math.ceil(b0);
- }
-
- public static BigDecimal ceil(BigDecimal b0) {
- return b0.setScale(0, BigDecimal.ROUND_CEILING);
- }
-
- /** SQL <code>CEIL</code> operator applied to byte values. */
- public static byte ceil(byte b0, byte b1) {
- return floor((byte) (b0 + b1 - 1), b1);
- }
-
- /** SQL <code>CEIL</code> operator applied to short values. */
- public static short ceil(short b0, short b1) {
- return floor((short) (b0 + b1 - 1), b1);
- }
-
- /** SQL <code>CEIL</code> operator applied to int values. */
- public static int ceil(int b0, int b1) {
- int r = b0 % b1;
- if (r > 0) {
- r -= b1;
- }
- return b0 - r;
- }
-
- /** SQL <code>CEIL</code> operator applied to long values. */
- public static long ceil(long b0, long b1) {
- return floor(b0 + b1 - 1, b1);
- }
-
- // temporary
- public static BigDecimal ceil(BigDecimal b0, int b1) {
- return ceil(b0, BigDecimal.valueOf(b1));
- }
-
- // temporary
- public static int ceil(int b0, BigDecimal b1) {
- return ceil(b0, b1.intValue());
- }
-
- public static BigDecimal ceil(BigDecimal b0, BigDecimal b1) {
- final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
- BigDecimal r = bigDecimals[1];
- if (r.signum() > 0) {
- r = r.subtract(b1);
- }
- return b0.subtract(r);
- }
-
- // ABS
-
- /** SQL <code>ABS</code> operator applied to byte values. */
- public static byte abs(byte b0) {
- return (byte) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to short values. */
- public static short abs(short b0) {
- return (short) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to int values. */
- public static int abs(int b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to long values. */
- public static long abs(long b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to float values. */
- public static float abs(float b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to double values. */
- public static double abs(double b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to BigDecimal values. */
- public static BigDecimal abs(BigDecimal b0) {
- return b0.abs();
- }
-
- // Helpers
-
- /** Helper for implementing MIN. Somewhat similar to LEAST operator. */
- public static <T extends Comparable<T>> T lesser(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- /** LEAST operator. */
- public static <T extends Comparable<T>> T least(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- public static boolean greater(boolean b0, boolean b1) {
- return b0 || b1;
- }
-
- public static boolean lesser(boolean b0, boolean b1) {
- return b0 && b1;
- }
-
- public static byte greater(byte b0, byte b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static byte lesser(byte b0, byte b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static char greater(char b0, char b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static char lesser(char b0, char b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static short greater(short b0, short b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static short lesser(short b0, short b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static int greater(int b0, int b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static int lesser(int b0, int b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static long greater(long b0, long b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static long lesser(long b0, long b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static float greater(float b0, float b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static float lesser(float b0, float b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static double greater(double b0, double b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static double lesser(double b0, double b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */
- public static <T extends Comparable<T>> T greater(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** GREATEST operator. */
- public static <T extends Comparable<T>> T greatest(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** Boolean comparison. */
- public static int compare(boolean x, boolean y) {
- return x == y ? 0 : x ? 1 : -1;
- }
-
- /** CAST(FLOAT AS VARCHAR). */
- public static String toString(float x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal = new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DOUBLE AS VARCHAR). */
- public static String toString(double x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal = new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DECIMAL AS VARCHAR). */
- public static String toString(BigDecimal x) {
- final String s = x.toString();
- if (s.startsWith("0")) {
- // we want ".1" not "0.1"
- return s.substring(1);
- } else if (s.startsWith("-0")) {
- // we want "-.1" not "-0.1"
- return "-" + s.substring(2);
- } else {
- return s;
- }
- }
-
- /** CAST(BOOLEAN AS VARCHAR). */
- public static String toString(boolean x) {
- // Boolean.toString returns lower case -- no good.
- return x ? "TRUE" : "FALSE";
- }
-
- @NonDeterministic
- private static Object cannotConvert(Object o, Class toType) {
- throw new RuntimeException("Cannot convert " + o + " to " + toType);
- }
-
- /** CAST(VARCHAR AS BOOLEAN). */
- public static boolean toBoolean(String s) {
- s = trim_(s, true, true, ' ');
- if (s.equalsIgnoreCase("TRUE")) {
- return true;
- } else if (s.equalsIgnoreCase("FALSE")) {
- return false;
- } else {
- throw new RuntimeException("Invalid character for cast");
- }
- }
-
- public static boolean toBoolean(Number number) {
- return !number.equals(0);
- }
-
- public static boolean toBoolean(Object o) {
- return o instanceof Boolean ? (Boolean) o : o instanceof Number ? toBoolean((Number) o) : o instanceof String ? toBoolean((String) o) : (Boolean) cannotConvert(o, boolean.class);
- }
-
- // Don't need parseByte etc. - Byte.parseByte is sufficient.
-
- public static byte toByte(Object o) {
- return o instanceof Byte ? (Byte) o : o instanceof Number ? toByte((Number) o) : Byte.parseByte(o.toString());
- }
-
- public static byte toByte(Number number) {
- return number.byteValue();
- }
-
- public static char toChar(String s) {
- return s.charAt(0);
- }
-
- public static Character toCharBoxed(String s) {
- return s.charAt(0);
- }
-
- public static short toShort(String s) {
- return Short.parseShort(s.trim());
- }
-
- public static short toShort(Number number) {
- return number.shortValue();
- }
-
- public static short toShort(Object o) {
- return o instanceof Short ? (Short) o : o instanceof Number ? toShort((Number) o) : o instanceof String ? toShort((String) o) : (Short) cannotConvert(o, short.class);
- }
-
- public static int toInt(java.util.Date v) {
- return toInt(v, LOCAL_TZ);
- }
-
- public static int toInt(java.util.Date v, TimeZone timeZone) {
- return (int) (toLong(v, timeZone) / DateTimeUtils.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.util.Date v) {
- return v == null ? null : toInt(v);
- }
-
- public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) {
- return v == null ? null : toInt(v, timeZone);
- }
-
- public static long toLong(Date v) {
- return toLong(v, LOCAL_TZ);
- }
-
- public static int toInt(java.sql.Time v) {
- return (int) (toLong(v) % DateTimeUtils.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.sql.Time v) {
- return v == null ? null : toInt(v);
- }
-
- public static int toInt(String s) {
- return Integer.parseInt(s.trim());
- }
-
- public static int toInt(Number number) {
- return number.intValue();
- }
-
- public static int toInt(Object o) {
- return o instanceof Integer ? (Integer) o : o instanceof Number ? toInt((Number) o) : o instanceof String ? toInt((String) o) : (Integer) cannotConvert(o, int.class);
- }
-
- public static long toLong(Timestamp v) {
- return toLong(v, LOCAL_TZ);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static long toLong(java.util.Date v, TimeZone timeZone) {
- final long time = v.getTime();
- return time + timeZone.getOffset(time);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static Long toLongOptional(java.util.Date v) {
- return v == null ? null : toLong(v, LOCAL_TZ);
- }
-
- public static Long toLongOptional(Timestamp v, TimeZone timeZone) {
- if (v == null) {
- return null;
- }
- return toLong(v, LOCAL_TZ);
- }
-
- public static long toLong(String s) {
- if (s.startsWith("199") && s.contains(":")) {
- return Timestamp.valueOf(s).getTime();
- }
- return Long.parseLong(s.trim());
- }
-
- public static long toLong(Number number) {
- return number.longValue();
- }
-
- public static long toLong(Object o) {
- return o instanceof Long ? (Long) o : o instanceof Number ? toLong((Number) o) : o instanceof String ? toLong((String) o) : (Long) cannotConvert(o, long.class);
- }
-
- public static float toFloat(String s) {
- return Float.parseFloat(s.trim());
- }
-
- public static float toFloat(Number number) {
- return number.floatValue();
- }
-
- public static float toFloat(Object o) {
- return o instanceof Float ? (Float) o : o instanceof Number ? toFloat((Number) o) : o instanceof String ? toFloat((String) o) : (Float) cannotConvert(o, float.class);
- }
-
- public static double toDouble(String s) {
- return Double.parseDouble(s.trim());
- }
-
- public static double toDouble(Number number) {
- return number.doubleValue();
- }
-
- public static double toDouble(Object o) {
- return o instanceof Double ? (Double) o : o instanceof Number ? toDouble((Number) o) : o instanceof String ? toDouble((String) o) : (Double) cannotConvert(o, double.class);
- }
-
- public static BigDecimal toBigDecimal(String s) {
- return new BigDecimal(s.trim());
- }
-
- public static BigDecimal toBigDecimal(Number number) {
- // There are some values of "long" that cannot be represented as "double".
- // Not so "int". If it isn't a long, go straight to double.
- return number instanceof BigDecimal ? (BigDecimal) number : number instanceof BigInteger ? new BigDecimal((BigInteger) number) : number instanceof Long ? new BigDecimal(number.longValue()) : new BigDecimal(number.doubleValue());
- }
-
- public static BigDecimal toBigDecimal(Object o) {
- return o instanceof Number ? toBigDecimal((Number) o) : toBigDecimal(o.toString());
- }
-
- // Don't need shortValueOf etc. - Short.valueOf is sufficient.
-
- /** Helper for CAST(... AS VARCHAR(maxLength)). */
- public static String truncate(String s, int maxLength) {
- return s == null ? null : s.length() > maxLength ? s.substring(0, maxLength) : s;
- }
-
- /** Helper for CAST(... AS VARBINARY(maxLength)). */
- public static ByteString truncate(ByteString s, int maxLength) {
- return s == null ? null : s.length() > maxLength ? s.substring(0, maxLength) : s;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(String seek, String s) {
- return s.indexOf(seek) + 1;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(ByteString seek, ByteString s) {
- return s.indexOf(seek) + 1;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long round(long v, long x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long truncate(long v, long x) {
- long remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int round(int v, int x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int truncate(int v, int x) {
- int remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** SQL {@code CURRENT_TIMESTAMP} function. */
- @NonDeterministic
- public static long currentTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root);
- }
-
- /** SQL {@code CURRENT_TIME} function. */
- @NonDeterministic
- public static int currentTime(DataContext root) {
- int time = (int) (currentTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY);
- if (time < 0) {
- time += DateTimeUtils.MILLIS_PER_DAY;
- }
- return time;
- }
-
- /** SQL {@code CURRENT_DATE} function. */
- @NonDeterministic
- public static int currentDate(DataContext root) {
- final long timestamp = currentTimestamp(root);
- int date = (int) (timestamp / DateTimeUtils.MILLIS_PER_DAY);
- final int time = (int) (timestamp % DateTimeUtils.MILLIS_PER_DAY);
- if (time < 0) {
- --date;
- }
- return date;
- }
-
- /** SQL {@code LOCAL_TIMESTAMP} function. */
- @NonDeterministic
- public static long localTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root);
- }
-
- /** SQL {@code LOCAL_TIME} function. */
- @NonDeterministic
- public static int localTime(DataContext root) {
- return (int) (localTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY);
- }
-
- /** Helper for "array element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object arrayItem(List list, int item) {
- if (item < 1 || item > list.size()) {
- return null;
- }
- return list.get(item - 1);
- }
-
- /** Helper for "map element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object mapItem(Map map, Object item) {
- return map.get(item);
- }
-
- /** Implements the {@code [ ... ]} operator on an object whose type is not
- * known until runtime.
- */
- public static Object item(Object object, Object index) {
- if (object instanceof Map) {
- return ((Map) object).get(index);
- }
- if (object instanceof List && index instanceof Number) {
- List list = (List) object;
- return list.get(((Number) index).intValue());
- }
- return null;
- }
-
- /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isTrue(Boolean b) {
- return b != null && b;
- }
-
- /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isNotFalse(Boolean b) {
- return b == null || b;
- }
-
- /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */
- public static Boolean not(Boolean b) {
- return (b == null) ? null : !b;
- }
-
- /** Converts a JDBC array to a list. */
- public static List arrayToList(final java.sql.Array a) {
- if (a == null) {
- return null;
- }
- try {
- return Primitive.asList(a.getArray());
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- /** Support the {@code CURRENT VALUE OF sequence} operator. */
- @NonDeterministic
- public static long sequenceCurrentValue(String key) {
- return getAtomicLong(key).get();
- }
-
- /** Support the {@code NEXT VALUE OF sequence} operator. */
- @NonDeterministic
- public static long sequenceNextValue(String key) {
- return getAtomicLong(key).incrementAndGet();
- }
-
- private static AtomicLong getAtomicLong(String key) {
- final Map<String, AtomicLong> map = THREAD_SEQUENCES.get();
- AtomicLong atomic = map.get(key);
- if (atomic == null) {
- atomic = new AtomicLong();
- map.put(key, atomic);
- }
- return atomic;
- }
-
- /** Support the SLICE function. */
- public static List slice(List list) {
- return list;
- }
-
- /** Support the ELEMENT function. */
- public static Object element(List list) {
- switch (list.size()) {
- case 0:
- return null;
- case 1:
- return list.get(0);
- default:
- throw new RuntimeException("more than one value");
- }
- }
-
- /** Returns a lambda that converts a list to an enumerable. */
- public static <E> Function1<List<E>, Enumerable<E>> listToEnumerable() {
- //noinspection unchecked
- return (Function1<List<E>, Enumerable<E>>) (Function1) LIST_AS_ENUMERABLE;
- }
-
-}
-
-// End SqlFunctions.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73abf6c/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
new file mode 100644
index 0000000..f88157c
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.calcite.sql2rel;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.tools.RelBuilder;
+
+/*
+ * OVERRIDE POINT:
+ * - disable the whole RelFieldTrimmer
+ */
+
+public class RelFieldTrimmer {
+
+ public RelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) {
+ }
+
+ public RelFieldTrimmer(SqlValidator validator, RelOptCluster cluster, RelFactories.ProjectFactory projectFactory, RelFactories.FilterFactory filterFactory, RelFactories.JoinFactory joinFactory, RelFactories.SemiJoinFactory semiJoinFactory, RelFactories.SortFactory sortFactory, RelFactories.AggregateFactory aggregateFactory, RelFactories.SetOpFactory setOpFactory) {
+ }
+
+ public RelNode trim(RelNode rootRel) {
+ return rootRel;
+ }
+
+}
[33/50] [abbrv] kylin git commit: minor,
enable some job configurations on mapper mem
Posted by sh...@apache.org.
minor, enable some job configurations on mapper mem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/18d4a5ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/18d4a5ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/18d4a5ec
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 18d4a5ecf83e951c35bb628430d5da8d9707f857
Parents: 6836815
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 11 12:43:23 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 11 12:45:44 2016 +0800
----------------------------------------------------------------------
build/conf/kylin_job_conf.xml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/18d4a5ec/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index f02004e..052021c 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -56,7 +56,6 @@
</property>
<!--Properties for calculating cube by splits (in-mem), with which each Mapper need more mem to hold a full cube segment -->
- <!--
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx2500m</value>
@@ -69,7 +68,6 @@
<name>mapreduce.task.io.sort.mb</name>
<value>200</value>
</property>
- -->
<property>
<name>mapred.task.timeout</name>
@@ -77,4 +75,4 @@
<description>Set task timeout to 1 hour</description>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
[42/50] [abbrv] kylin git commit: KYLIN-1286 fix java license files
Posted by sh...@apache.org.
KYLIN-1286 fix java license files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85c10686
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85c10686
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85c10686
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 85c10686c12f21cf2730893dc7369294256657af
Parents: a44a57c
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 11 16:26:38 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 11 16:26:38 2016 +0800
----------------------------------------------------------------------
.../storage/hbase/common/coprocessor/AggrKey.java | 18 ++++++++++++++++++
.../common/coprocessor/CoprocessorBehavior.java | 18 ++++++++++++++++++
.../hbase/common/coprocessor/FilterDecorator.java | 18 ++++++++++++++++++
.../storage/hbase/cube/v1/CubeTupleConverter.java | 18 ++++++++++++++++++
.../cube/v1/filter/generated/FilterProtosExt.java | 18 ++++++++++++++++++
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 18 ++++++++++++++++++
.../storage/hbase/cube/v2/CubeSegmentScanner.java | 18 ++++++++++++++++++
.../storage/hbase/cube/v2/CubeStorageQuery.java | 18 ++++++++++++++++++
.../cube/v2/SequentialCubeTupleIterator.java | 18 ++++++++++++++++++
.../endpoint/generated/CubeVisitProtos.java | 18 ++++++++++++++++++
.../endpoint/EndpointTupleConverter.java | 18 ++++++++++++++++++
.../ii/coprocessor/endpoint/LocalDictionary.java | 18 ++++++++++++++++++
.../coprocessor/endpoint/generated/IIProtos.java | 18 ++++++++++++++++++
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 18 ++++++++++++++++++
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 18 ++++++++++++++++++
.../storage/hbase/steps/KeyValueCreator.java | 18 ++++++++++++++++++
.../hbase/steps/SequenceFileCuboidWriter.java | 18 ++++++++++++++++++
.../storage/hbase/util/CubeMigrationCheckCLI.java | 18 ++++++++++++++++++
.../storage/hbase/util/HbaseStreamingInput.java | 18 ++++++++++++++++++
.../kylin/storage/hbase/util/RowCounterCLI.java | 18 ++++++++++++++++++
.../storage/hbase/util/ZookeeperJobLock.java | 18 ++++++++++++++++++
.../common/TimeConditionLiteralsReplacerTest.java | 18 ++++++++++++++++++
.../hbase/common/TsConditionEraserTest.java | 18 ++++++++++++++++++
23 files changed, 414 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java
index 3fbc69b..e071f44 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggrKey.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.common.coprocessor;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
index df81177..21dc0d1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.common.coprocessor;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 5208ba7..1030fbc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.common.coprocessor;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 4f674cb..dbd6a45 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v1;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
index 114517e..6266f17 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index fa32769..ef53cb7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index ee5ae96..80f8c67 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index df0bb84..85082ca 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v2;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index 023b5f8..bbc7370 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 1094f80..6e3e2bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
index 1073021..c60ce0b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
index 94e5b33..7524e50 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import org.apache.kylin.common.util.Dictionary;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
index 4f48a0c..916a0a0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/protobuf/II.proto
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 5d72756..f2aba0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.steps;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index f71d0f8..623274d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.steps;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
index 2f72b7d..c6a233b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.steps;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 8c2d5e4..801f8e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.steps;
import org.apache.commons.io.IOUtils;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 2762561..d3a85f0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.util;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index adf4419..b5c9b1d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 038e954..0bf2fc9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index d211206..729635b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.util;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
index f85bb8e..990a326 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.common;
import org.apache.kylin.dict.DictCodeSystem;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85c10686/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
index 6c6ed80..90f3370 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.kylin.storage.hbase.common;
import java.io.IOException;
[23/50] [abbrv] kylin git commit: KYLIN-1464 prevent
NullPointerException when CubeDesc init failed due to IllegalStateException
Posted by sh...@apache.org.
KYLIN-1464 prevent NullPointerException when CubeDesc init failed due to IllegalStateException
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed57c3b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed57c3b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed57c3b3
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: ed57c3b30c8680c2bcb73b6d8b889d05b952d596
Parents: 26233f7
Author: John Zhao <yu...@ebay.com>
Authored: Wed Mar 2 14:51:58 2016 -0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 10 09:27:10 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeInstance.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed57c3b3/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2862d4f..d89e736 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -125,7 +125,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@Override
public DataModelDesc getDataModelDesc() {
- return this.getDescriptor().getModel();
+ CubeDesc cubeDesc = this.getDescriptor();
+ if (cubeDesc != null) {
+ return cubeDesc.getModel();
+ } else {
+ return null;
+ }
}
public boolean isReady() {
[04/50] [abbrv] kylin git commit: KYLIN-1469 Find hive dependencies
from jvm if not exists in environment variables
Posted by sh...@apache.org.
KYLIN-1469 Find hive dependencies from jvm if not exists in environment variables
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b41c4460
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b41c4460
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b41c4460
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: b41c44600f801f80aa379494bd3be9bc77d5fa65
Parents: 8a3e0e0
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 4 14:02:28 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 4 14:02:28 2016 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 22 +++++++++++++++++++-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b41c4460/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index e4eee96..61983d5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -23,7 +23,7 @@ package org.apache.kylin.engine.mr.common;
*
*/
-import static org.apache.hadoop.util.StringUtils.formatTime;
+import static org.apache.hadoop.util.StringUtils.*;
import java.io.File;
import java.io.IOException;
@@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -217,6 +218,25 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
if (kylinDependency.length() > 0)
kylinDependency.append(",");
kylinDependency.append(filteredHive);
+ } else {
+
+ logger.info("No hive dependency jars set in the environment, will find them from jvm:");
+
+ try {
+ String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
+ kylinDependency.append(hiveExecJarPath).append(",");
+ logger.info("hive-exec jar file: " + hiveExecJarPath);
+
+ String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
+ kylinDependency.append(hiveHCatJarPath).append(",");
+ logger.info("hive-catalog jar file: " + hiveHCatJarPath);
+
+ String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
+ kylinDependency.append(hiveMetaStoreJarPath).append(",");
+ logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
+ } catch (ClassNotFoundException e) {
+ logger.error("Cannot found hive dependency jars: " + e);
+ }
}
// for KylinJobMRLibDir
http://git-wip-us.apache.org/repos/asf/kylin/blob/b41c4460/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 7b1a4be..18f6feb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -58,7 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
[43/50] [abbrv] kylin git commit: KYLIN-1286 fix java license files
Posted by sh...@apache.org.
KYLIN-1286 fix java license files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7377eda2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7377eda2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7377eda2
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 7377eda2253f7d3250781fbb50ca13af9243f3c5
Parents: 85c1068
Author: Li Yang <li...@apache.org>
Authored: Fri Mar 11 16:35:20 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Mar 11 16:35:20 2016 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/invertedindex/IITest.java | 18 ++++++++++++++++++
.../kylin/job/streaming/KafkaDataLoader.java | 18 ++++++++++++++++++
.../streaming/StreamingTableDataGenerator.java | 18 ++++++++++++++++++
3 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7377eda2/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index ac91efb..da25143 100644
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.job.hadoop.invertedindex;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7377eda2/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java b/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
index 4d9c0aa..31fc670 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.job.streaming;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7377eda2/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
index 4ce223e..734c982 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.job.streaming;
import java.util.HashMap;
[03/50] [abbrv] kylin git commit: minor,
UI fix kylin properties format
Posted by sh...@apache.org.
minor, UI fix kylin properties format
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a3e0e0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a3e0e0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a3e0e0e
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 8a3e0e0e869fd4ec52a847fa360ec3d3c3e5a5bf
Parents: daeaf08
Author: Jason <ji...@163.com>
Authored: Fri Mar 4 10:57:46 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Mar 4 10:58:24 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/kylin.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a3e0e0e/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 7c9919b..04681af 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -116,5 +116,5 @@ kylin.web.contact_mail=
#env DEV|QA|PROD
deploy.env=DEV
- kylin.web.hive.limit=20
+kylin.web.hive.limit=20
[05/50] [abbrv] kylin git commit: KYLIN-1471 - LIMIT after having
clause should not be pushed down to storage context
Posted by sh...@apache.org.
KYLIN-1471 - LIMIT after having clause should not be pushed down to storage context
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f44970d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f44970d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f44970d
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 2f44970d28318a21c16aaa8f28e844c5d88f5e3d
Parents: b9a3418
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 4 16:00:11 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 4 16:01:04 2016 +0800
----------------------------------------------------------------------
.../resources/query/sql_tableau/query29.sql | 29 ++++++++++++++++++++
.../apache/kylin/query/relnode/OLAPContext.java | 1 +
.../kylin/query/relnode/OLAPFilterRel.java | 2 ++
.../kylin/query/relnode/OLAPLimitRel.java | 21 ++++++++------
4 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/kylin-it/src/test/resources/query/sql_tableau/query29.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query29.sql b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
new file mode 100644
index 0000000..0858087
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *
+ FROM (
+ select test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(*) as TRANS_CNT
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ group by test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt
+ ) "TableauSQL"
+ LIMIT 1
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 6de1790..431328f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -103,6 +103,7 @@ public class OLAPContext {
public OLAPTableScan firstTableScan = null; // to be fact table scan except "select * from lookupTable"
public TupleInfo returnTupleInfo = null;
public boolean afterAggregate = false;
+ public boolean afterSkippedFilter = false;
public boolean afterJoin = false;
public boolean hasJoin = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 7b8bfdb..a847890 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -271,6 +271,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
// only translate where clause and don't translate having clause
if (!context.afterAggregate) {
translateFilter(context);
+ } else {
+ context.afterSkippedFilter = true;//having clause is skipped
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 572a5c7..82aa9de 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -73,16 +73,19 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
implementor.visitChild(getInput(), this);
this.columnRowType = buildColumnRowType();
-
this.context = implementor.getContext();
- Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
- int limit = limitValue.intValue();
- this.context.storageContext.setLimit(limit);
- this.context.limit = limit;
- if(localOffset != null) {
- Number offsetValue = (Number) (((RexLiteral) localOffset).getValue());
- int offset = offsetValue.intValue();
- this.context.storageContext.setOffset(offset);
+
+ if (!context.afterSkippedFilter) {
+ Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
+ int limit = limitValue.intValue();
+ this.context.storageContext.setLimit(limit);
+ this.context.limit = limit;
+
+ if (localOffset != null) {
+ Number offsetValue = (Number) (((RexLiteral) localOffset).getValue());
+ int offset = offsetValue.intValue();
+ this.context.storageContext.setOffset(offset);
+ }
}
}
[02/50] [abbrv] kylin git commit: KYLIN-1466 Update kylin.sh and
bring denpendencies to runnable classes
Posted by sh...@apache.org.
KYLIN-1466 Update kylin.sh and bring denpendencies to runnable classes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/daeaf084
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/daeaf084
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/daeaf084
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: daeaf08444ebd86a9de4e8addafa7541c25523be
Parents: 2d4922d
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 3 23:26:32 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 3 23:26:55 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/daeaf084/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index a21928e..f4d9fac 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -206,7 +206,7 @@ then
export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*:$hive_dependency:${HBASE_CLASSPATH}
- exec hbase -Dlog4j.configuration=kylin-log4j.properties "$@"
+ exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=kylin-log4j.properties "$@"
else
echo "usage: kylin.sh start or kylin.sh stop"
[48/50] [abbrv] kylin git commit: [maven-release-plugin] prepare for
next development iteration
Posted by sh...@apache.org.
[maven-release-plugin] prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bdba620
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bdba620
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bdba620
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 5bdba6208a64653357657b1302179cb4c714414b
Parents: 64d0201
Author: Yang Li <li...@apache.org>
Authored: Sat Mar 12 12:56:00 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Mar 12 12:56:00 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
engine-streaming/pom.xml | 2 +-
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml | 4 ++--
query/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
20 files changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 20f02a0..84f9189 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 6ca1eae..93f52b7 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 776ba76..3e5b138 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 97af425..9fde78a 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index d518ecf..9718b67 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index a17418c..ffe09b0 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 89d85f2..caf5246 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index b529488..9bb78ac 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 088641b..0d0f7b8 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 9cce44a..9998f66 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index ee1f498..9bc3464 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 2082329..5007629 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index c1311db..6af40d3 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 5b42622..e6cf589 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8d95b7..f878d8b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
<name>Kylin:HadoopOLAPEngine</name>
<url>http://kylin.apache.org</url>
@@ -167,7 +167,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/kylin.git</url>
- <tag>kylin-1.5.0</tag>
+ <tag>apache-kylin-2.1</tag>
</scm>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index c1b3213..3fb407c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 253ce69..5442f7f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 7a0c560..46c016d 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 4ed14e5..86a0e98 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdba620/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index c15d617..8d3d814 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.0</version>
+ <version>1.5.0-SNAPSHOT</version>
</parent>
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.1 API
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 7ac4be1..dddb2ad 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -129,7 +129,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -150,7 +150,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
GTScanRequest scanReq = GTScanRequest.serializer.deserialize(ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getGtScanRequest())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
index 9c96f21..a277eeb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
@@ -24,7 +24,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
@@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.IIDeployCoprocessorCLI;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
@@ -47,7 +49,7 @@ public class IICreateHTableJob extends AbstractHadoopJob {
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
options.addOption(OPTION_II_NAME);
options.addOption(OPTION_HTABLE_NAME);
@@ -63,10 +65,11 @@ public class IICreateHTableJob extends AbstractHadoopJob {
Configuration conf = HBaseConfiguration.create(getConf());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// check if the table already exists
- admin = new HBaseAdmin(conf);
- if (admin.tableExists(tableName)) {
- if (admin.isTableEnabled(tableName)) {
+ admin = conn.getAdmin();
+ if (admin.tableExists(TableName.valueOf(tableName))) {
+ if (admin.isTableEnabled(TableName.valueOf(tableName))) {
logger.info("Table " + tableName + " already exists and is enabled, no need to create.");
return 0;
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
index 5afb62d..dd65f9a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
@@ -20,7 +20,7 @@ package org.apache.kylin.storage.hbase.ii;
import java.util.ArrayList;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -55,8 +55,8 @@ public class InvertedIndexStorageQuery implements ICachableStorageQuery {
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
String tableName = seg.getStorageLocationIdentifier();
- //HConnection is cached, so need not be closed
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ //Connection is cached, so need not be closed
+ Connection conn = HBaseConnection.get(context.getConnUrl());
try {
dataIterator = new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn, returnTupleInfo);
return dataIterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 4ec421b..8da8e59 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -31,8 +31,9 @@ import javax.annotation.Nullable;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SerializationUtils;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -93,16 +94,16 @@ public class EndpointTupleIterator implements ITupleIterator {
private Iterator<List<IIProtos.IIResponseInternal.IIRow>> regionResponsesIterator = null;
private ITupleIterator tupleIterator = null;
- private HTableInterface table = null;
+ private Table table = null;
private TblColRef partitionCol;
private long lastDataTime = -1;
private int rowsInAllMetric = 0;
- public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn, TupleInfo returnTupleInfo) throws Throwable {
+ public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, Connection conn, TupleInfo returnTupleInfo) throws Throwable {
String tableName = segment.getStorageLocationIdentifier();
- table = conn.getTable(tableName);
+ table = conn.getTable(TableName.valueOf(tableName));
factTableName = segment.getIIDesc().getFactTableName();
if (rootFilter == null) {
@@ -290,7 +291,7 @@ public class EndpointTupleIterator implements ITupleIterator {
return request;
}
- private Collection<IIProtos.IIResponse> getResults(final IIProtos.IIRequest request, HTableInterface table) throws Throwable {
+ private Collection<IIProtos.IIResponse> getResults(final IIProtos.IIRequest request, Table table) throws Throwable {
Map<byte[], IIProtos.IIResponse> results = table.coprocessorService(IIProtos.RowsService.class, null, null, new Batch.Call<IIProtos.RowsService, IIProtos.IIResponse>() {
public IIProtos.IIResponse call(IIProtos.RowsService rowsService) throws IOException {
ServerRpcController controller = new ServerRpcController();
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index af7b993..4724188 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -89,7 +89,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
if (request.hasTsRange()) {
Range<Long> tsRange = (Range<Long>) SerializationUtils.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getTsRange()));
- byte[] regionStartKey = region.getStartKey();
+ byte[] regionStartKey = region.getRegionInfo().getStartKey();
if (!ArrayUtils.isEmpty(regionStartKey)) {
shard = BytesUtil.readUnsigned(regionStartKey, 0, IIKeyValueCodec.SHARD_LEN);
} else {
@@ -143,7 +143,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
HRegion region = null;
try {
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
innerScanner = region.getScanner(prepareScan(request, region));
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index f2aba0a..be32903 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
@@ -70,7 +72,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableUser, cubeInstance.getOwner());
Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -117,7 +120,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -126,7 +129,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} catch (Exception e) {
logger.error("Failed to create HTable", e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 735f967..9dc9715 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -27,17 +27,18 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HiveCmdBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index ddc868d..5063e00 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -36,7 +36,7 @@ package org.apache.kylin.storage.hbase.steps;
import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -63,7 +63,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -72,7 +72,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -131,7 +131,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index fa62a62..cf5c9d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -28,7 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -55,7 +56,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
try {
CubeSegment cubeSegment = (CubeSegment) buildable;
- final HTableInterface hTable;
+ final Table hTable;
hTable = createHTable(cubeSegment);
List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -87,10 +88,10 @@ public class HBaseStreamingOutput implements IStreamingOutput {
}
}
- private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException {
+ private Table createHTable(final CubeSegment cubeSegment) throws IOException {
final String hTableName = cubeSegment.getStorageLocationIdentifier();
CubeHTableUtil.createHTable(cubeSegment, null);
- final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+ final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!");
return hTable;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index a4a8a35..dfde2dd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,17 +24,17 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 957e04b..51ecc8f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,13 +21,15 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,8 +54,9 @@ public class CleanHtableCLI extends AbstractHadoopJob {
}
private void clean() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -64,7 +67,7 @@ public class CleanHtableCLI extends AbstractHadoopJob {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 09aab48..f15100d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -73,7 +73,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -117,8 +117,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConfiguration.create();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -143,6 +143,8 @@ public class CubeMigrationCLI {
}
checkMigrationSuccess(dstConfig, cubeName, true);
+
+ IOUtils.closeQuietly(hbaseAdmin);
}
public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
@@ -311,10 +313,10 @@ public class CubeMigrationCLI {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
logger.info("CHANGE_HTABLE_HOST is completed");
break;
}
@@ -425,11 +427,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -445,11 +447,10 @@ public class CubeMigrationCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
+ put.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), CellUtil.cloneValue(cell));
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -476,10 +477,10 @@ public class CubeMigrationCLI {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
break;
}
case COPY_FILE_IN_META: {
@@ -509,13 +510,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index d3a85f0..b735864 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -27,13 +27,16 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +63,7 @@ public class CubeMigrationCheckCLI {
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -128,9 +131,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConfiguration.create();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -187,10 +189,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable "+sepNameList[0]+"belonging to cube "+sepNameList[1]+" from "+desc.getValue(IRealizationConstants.HTableTag)+" to "+dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
}else{
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 5bca721..749f8aa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
@@ -73,7 +75,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar = new File(args[0]).getAbsolutePath();
logger.info("Identify coprocessor jar " + localCoprocessorJar);
@@ -154,10 +157,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -180,9 +183,9 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static void resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
logger.info("Unset coprocessor on " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -203,13 +206,13 @@ public class DeployCoprocessorCLI {
desc.removeCoprocessor(IIEndpointClassOld);
}
addCoprocessorOnHTable(desc, hdfsCoprocessorJar);
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
}
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(Admin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processed = new ArrayList<String>();
for (String tableName : tableNames) {
@@ -320,7 +323,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index c55bde4..ad5b647 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -26,8 +26,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
@@ -232,9 +233,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,7 +255,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 8a88b6d..760ed3a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,9 +28,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 621909a..f5b315d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -29,10 +29,15 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,8 +83,8 @@ public class HBaseClean extends AbstractHadoopJob {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -94,12 +99,12 @@ public class HBaseClean extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 346c3a2..58aa8fd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -20,22 +20,11 @@
package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -57,30 +46,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -104,7 +94,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- hBaseAdmin.close();
+ admin.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index ffb1e25..f2578c0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -26,11 +26,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.kylin.storage.hbase.HBaseConnection;
public class HBaseUsage {
@@ -42,8 +46,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index b5c9b1d..79f9fb3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,16 +32,18 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,11 +59,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -118,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -134,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -169,8 +171,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -205,7 +207,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -223,8 +225,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConfiguration.create());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index 239adcf..095743f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -27,9 +27,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,8 +74,8 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob {
}
private void alter() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index f0618c9..b7303a5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -31,10 +31,15 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,7 +91,8 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -105,12 +111,12 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 58ef7cb..b039a03 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -21,9 +21,10 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -57,12 +58,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 0bf2fc9..dbeab73 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -23,9 +23,10 @@ import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.kylin.common.util.Bytes;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index c010d51..fd39ee6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -35,7 +35,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -52,6 +54,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,7 +107,8 @@ public class StorageCleanupJob extends AbstractHadoopJob {
IIManager iiManager = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -148,12 +152,12 @@ public class StorageCleanupJob extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 23357f5..3fec98c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -26,9 +26,12 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -39,7 +42,6 @@ import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import com.google.common.collect.Lists;
@@ -52,14 +54,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -181,9 +184,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index cd4e33d..9585575 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -29,12 +29,10 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -233,15 +231,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -310,6 +301,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -326,16 +322,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dffbbf3b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
[45/50] [abbrv] kylin git commit: minor, fix dev-support script
Posted by sh...@apache.org.
minor, fix dev-support script
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4aeb76b5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4aeb76b5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4aeb76b5
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 4aeb76b5613caba382a0d855bcb19fdb36ce7122
Parents: bccdd4f
Author: Yang Li <li...@apache.org>
Authored: Sat Mar 12 09:07:06 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Mar 12 09:07:06 2016 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4aeb76b5/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
index 0f81d2e..f0d27e4 100644
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -21,5 +21,5 @@ dir=$(dirname ${0})
cd ${dir}
cd ..
-mvn clean install -DskipTests | tee mci.log
-mvn verify -Dhdp.version=2.2.4.2-2 -fae | tee mvnverify.log
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify -Dhdp.version=2.2.4.2-2 -fae 2>&1 | tee mvnverify.log
[31/50] [abbrv] kylin git commit: KYLIN-1431 load streaming table ui
update add cluster info when add table
Posted by sh...@apache.org.
KYLIN-1431 load streaming table ui update add cluster info when add table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/890577da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/890577da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/890577da
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 890577dad510587400640b40e16810e5caa43df2
Parents: 3569b22
Author: Jason <ji...@163.com>
Authored: Wed Mar 9 19:01:42 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Mar 11 10:58:35 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 68 -----------
.../kylin/rest/service/StreamingService.java | 4 +-
webapp/app/js/controllers/cubeEdit.js | 82 ++++++--------
webapp/app/js/controllers/cubeRefresh.js | 1 -
webapp/app/js/controllers/cubeSchema.js | 2 +-
webapp/app/js/controllers/sourceMeta.js | 82 ++++++++++++--
webapp/app/js/controllers/streamingConfig.js | 8 ++
webapp/app/js/model/modelsManager.js | 1 +
webapp/app/less/app.less | 4 +
.../partials/cubeDesigner/streamingConfig.html | 16 +--
.../app/partials/tables/loadStreamingTable.html | 89 +++++++++++++++
webapp/app/partials/tables/table_load.html | 112 +++++--------------
12 files changed, 242 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e60f330..bc00795 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -356,74 +356,6 @@ public class CubeController extends BasicController {
throw new InternalErrorException(e.getLocalizedMessage(), e);
}
- boolean createStreamingConfigSuccess = false, createKafkaConfigSuccess = false;
- StreamingConfig streamingConfig = null;
- KafkaConfig kafkaConfig = null;
-
- boolean isStreamingCube = cubeRequest.getStreamingCube() != null && cubeRequest.getStreamingCube().equals("true");
- try {
- //streaming Cube
- if (isStreamingCube) {
- streamingConfig = deserializeStreamingDesc(cubeRequest);
- kafkaConfig = deserializeKafkaDesc(cubeRequest);
- // validate before create, rollback when error
- if (kafkaConfig == null) {
- cubeRequest.setMessage("No KafkaConfig info defined.");
- return cubeRequest;
- }
- if (streamingConfig == null) {
- cubeRequest.setMessage("No StreamingConfig info defined.");
- return cubeRequest;
- }
-
- try {
- streamingConfig.setUuid(UUID.randomUUID().toString());
- streamingService.createStreamingConfig(streamingConfig);
- createStreamingConfigSuccess = true;
- } catch (IOException e) {
- logger.error("Failed to save StreamingConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to save StreamingConfig: " + e.getLocalizedMessage());
- }
- try {
- kafkaConfig.setUuid(UUID.randomUUID().toString());
- kafkaConfigService.createKafkaConfig(kafkaConfig);
- createKafkaConfigSuccess = true;
- } catch (IOException e) {
- logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
- throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage());
- }
-
- }
- } finally {
- //rollback if failed
- if (isStreamingCube) {
- if (createStreamingConfigSuccess == false || createKafkaConfigSuccess == false) {
- try {
- cubeService.deleteCube(cubeInstance);
- } catch (Exception ex) {
- throw new InternalErrorException("Failed to rollback on delete cube. " + " Caused by: " + ex.getMessage(), ex);
- }
- if (createStreamingConfigSuccess == true) {
- try {
- streamingService.dropStreamingConfig(streamingConfig);
- } catch (IOException e) {
- throw new InternalErrorException("Failed to create cube, and StreamingConfig created and failed to delete: " + e.getLocalizedMessage());
- }
- }
- if (createKafkaConfigSuccess == true) {
- try {
- kafkaConfigService.dropKafkaConfig(kafkaConfig);
- } catch (IOException e) {
- throw new InternalErrorException("Failed to create cube, and KafkaConfig created and failed to delete: " + e.getLocalizedMessage());
- }
- }
-
- }
-
- }
-
- }
-
cubeRequest.setUuid(desc.getUuid());
cubeRequest.setSuccessful(true);
return cubeRequest;
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
index a0473e9..9627e8f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
@@ -45,7 +45,9 @@ public class StreamingService extends BasicService {
streamingConfigs = getStreamingManager().listAllStreaming();
} else {
StreamingConfig config = getStreamingManager().getConfig(table);
- streamingConfigs.add(config);
+ if(config!=null){
+ streamingConfigs.add(config);
+ }
}
return streamingConfigs;
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 6f0df38..24bdf8d 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -171,24 +171,25 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
if (!modelsManager.getModels().length) {
ModelDescService.query({model_name: $scope.cubeMetaFrame.model_name}, function (_model) {
$scope.metaModel.model = _model;
+
+ StreamingService.getConfig({table:$scope.metaModel.model.fact_table}, function (kfkConfigs) {
+ if(!!kfkConfigs[0]){
+ $scope.cubeState.isStreaming = true;
+ }
+ else{
+ return;
+ }
+ $scope.streamingMeta = kfkConfigs[0];
+ StreamingService.getKfkConfig({kafkaConfigName:$scope.streamingMeta.name}, function (streamings) {
+ $scope.kafkaMeta = streamings[0];
+ })
+ })
});
}
- $scope.metaModel.model = modelsManager.getModel($scope.cubeMetaFrame.model_name);
$scope.state.cubeSchema = angular.toJson($scope.cubeMetaFrame, true);
- StreamingService.getConfig({cubeName:$scope.cubeMetaFrame.name}, function (kfkConfigs) {
- if(!!kfkConfigs[0]){
- $scope.cubeState.isStreaming = true;
- }
- else{
- return;
- }
- $scope.streamingMeta = kfkConfigs[0];
- StreamingService.getKfkConfig({kafkaConfigName:$scope.streamingMeta.name}, function (streamings) {
- $scope.kafkaMeta = streamings[0];
- })
- })
+
}
});
@@ -732,41 +733,32 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
});
- $scope.streamingCfg = {
- parseTsColumn:"{{}}",
- columnOptions:[]
- }
//dimensions options is depend on the model input when add cube
- $scope.$watch('cubeMetaFrame.model_name', function (newValue, oldValue) {
- if (!newValue) {
- return;
- }
- $scope.metaModel.model = modelsManager.getModel(newValue);
+ //$scope.$watch('cubeMetaFrame.model_name', function (newValue, oldValue) {
+ // if (!newValue) {
+ // return;
+ // }
+ // $scope.metaModel.model = modelsManager.getModel(newValue);
+ //
+ // if(!$scope.metaModel.model){
+ // return;
+ // }
+ //
+ // var factTable = $scope.metaModel.model.fact_table;
+ // var cols = $scope.getColumnsByTable(factTable);
+ //
+ // for(var i=0;i<cols.length;i++){
+ // var col = cols[i];
+ // if(col.datatype === "timestamp"){
+ // $scope.streamingCfg.columnOptions.push(col.name);
+ // }
+ // }
+ // $scope.kafkaMeta.parserProperties = "tsColName=' ';formatTs=TRUE";
+ //
+ //
+ //});
- if(!$scope.metaModel.model){
- return;
- }
-
- var factTable = $scope.metaModel.model.fact_table;
- var cols = $scope.getColumnsByTable(factTable);
-
- for(var i=0;i<cols.length;i++){
- var col = cols[i];
- if(col.datatype === "timestamp"){
- $scope.streamingCfg.columnOptions.push(col.name);
- }
- }
- $scope.kafkaMeta.parserProperties = "tsColName=' ';formatTs=TRUE";
-
- });
-
- $scope.streamingTsColUpdate = function(){
- if(!$scope.streamingCfg.parseTsColumn){
- $scope.streamingCfg.parseTsColumn = ' ';
- }
- $scope.kafkaMeta.parserProperties = "tsColName="+$scope.streamingCfg.parseTsColumn+";formatTs=TRUE";
- }
$scope.$on('DimensionsEdited', function (event) {
if ($scope.cubeMetaFrame) {
reGenerateRowKey();
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/controllers/cubeRefresh.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeRefresh.js b/webapp/app/js/controllers/cubeRefresh.js
index d0de50a..ac4b7f7 100644
--- a/webapp/app/js/controllers/cubeRefresh.js
+++ b/webapp/app/js/controllers/cubeRefresh.js
@@ -23,7 +23,6 @@ KylinApp.controller('CubeRefreshCtrl', function ($scope, $modal,cubeConfig,MetaM
//edit model
if($scope.state.mode==="edit") {
- $scope.metaModel = MetaModel;
if(!$scope.cubeMetaFrame.auto_merge_time_ranges){
$scope.cubeMetaFrame.auto_merge_time_ranges = [];
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index d446730..d4f1e1c 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -34,7 +34,7 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
if (UserService.hasRole("ROLE_ADMIN")) {
$scope.wizardSteps.push({title: 'Advanced Setting', src: 'partials/cubeDesigner/advanced_settings.html', isComplete: false,form:'cube_setting_form'});
}
- $scope.wizardSteps.push({title: 'Streaming', src: 'partials/cubeDesigner/streamingConfig.html', isComplete: false,form:'cube_streaming_form'});
+ //$scope.wizardSteps.push({title: 'Streaming', src: 'partials/cubeDesigner/streamingConfig.html', isComplete: false,form:'cube_streaming_form'});
$scope.wizardSteps.push({title: 'Overview', src: 'partials/cubeDesigner/overview.html', isComplete: false,form:null});
$scope.curStep = $scope.wizardSteps[0];
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index 69f1a44..472b18b 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -422,7 +422,72 @@ KylinApp
});
};
- var StreamingSourceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, scope, tableConfig,cubeConfig) {
+ var StreamingSourceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, scope, tableConfig,cubeConfig,StreamingModel) {
+
+ $scope.cubeState={
+ "isStreaming": false
+ }
+ $scope.state={
+ 'mode':'edit'
+ }
+
+ $scope.streamingMeta = StreamingModel.createStreamingConfig();
+ $scope.kafkaMeta = StreamingModel.createKafkaConfig();
+
+
+ $scope.steps = {
+ curStep:1
+ };
+
+ $scope.streamingCfg = {
+ parseTsColumn:"{{}}",
+ columnOptions:[]
+ }
+
+ $scope.previewStep = function(){
+ $scope.steps.curStep--;
+ }
+
+ $scope.nextStep = function(){
+ //check form
+ $scope.form['setStreamingSchema'].$sbumitted = true;
+ if(!$scope.streaming.sourceSchema||$scope.streaming.sourceSchema===""){
+ return;
+ }
+
+ if(!$scope.table.name||$scope.table.name===""){
+ return;
+ }
+
+ $scope.prepareNextStep();
+
+ if(!$scope.rule.timestampColumnExist){
+ return;
+ }
+
+ $scope.steps.curStep++;
+ }
+
+ $scope.prepareNextStep = function(){
+ $scope.streamingCfg.columnOptions = [];
+ angular.forEach($scope.columnList,function(column,$index){
+ if (column.checked == "Y" && column.fromSource=="Y" && column.type == "timestamp") {
+ $scope.streamingCfg.columnOptions.push(column.name);
+ $scope.rule.timestampColumnExist = true;
+ }
+ })
+
+ if($scope.streamingCfg.columnOptions.length==1){
+ $scope.streamingCfg.parseTsColumn = $scope.streamingCfg.columnOptions[0];
+ $scope.kafkaMeta.parserProperties = "tsColName="+$scope.streamingCfg.parseTsColumn;
+ }
+ if($scope.kafkaMeta.parserProperties!==''){
+ $scope.state.isParserHeaderOpen = false;
+ }else{
+ $scope.state.isParserHeaderOpen = true;
+ }
+ }
+
$scope.projectName = projectName;
$scope.tableConfig = tableConfig;
$scope.cubeConfig = cubeConfig;
@@ -433,7 +498,8 @@ KylinApp
$scope.table = {
name: '',
- sourceValid:false
+ sourceValid:false,
+ schemaChecked:false
}
$scope.cancel = function () {
@@ -447,6 +513,7 @@ KylinApp
$scope.columnList = [];
$scope.streamingOnChange = function () {
+ $scope.table.schemaChecked = true;
console.log($scope.streaming.sourceSchema);
try {
$scope.streaming.parseResult = JSON.parse($scope.streaming.sourceSchema);
@@ -525,17 +592,9 @@ KylinApp
$scope.form={};
$scope.rule={
- 'timestampColumnConflict':false
+ 'timestampColumnExist':false
}
$scope.syncStreamingSchema = function () {
- $scope.form['setStreamingSchema'].$sbumitted = true;
- if(!$scope.streaming.sourceSchema||$scope.streaming.sourceSchema===""){
- return;
- }
-
- if(!$scope.table.name||$scope.table.name===""){
- return;
- }
var columns = [];
angular.forEach($scope.columnList,function(column,$index){
@@ -552,6 +611,7 @@ KylinApp
$scope.tableData = {
"name": $scope.table.name,
+ "source_type":1,
"columns": columns,
'database':'Default'
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/controllers/streamingConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/streamingConfig.js b/webapp/app/js/controllers/streamingConfig.js
index 53b30e0..b0a1ebd 100644
--- a/webapp/app/js/controllers/streamingConfig.js
+++ b/webapp/app/js/controllers/streamingConfig.js
@@ -72,4 +72,12 @@ KylinApp.controller('streamingConfigCtrl', function ($scope, $q, $routeParams, $
delete cluster.newBroker;
}
+
+ $scope.streamingTsColUpdate = function(){
+ if(!$scope.streamingCfg.parseTsColumn){
+ $scope.streamingCfg.parseTsColumn = ' ';
+ }
+ $scope.kafkaMeta.parserProperties = "tsColName="+$scope.streamingCfg.parseTsColumn;
+ }
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/js/model/modelsManager.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/modelsManager.js b/webapp/app/js/model/modelsManager.js
index e117053..03f0480 100644
--- a/webapp/app/js/model/modelsManager.js
+++ b/webapp/app/js/model/modelsManager.js
@@ -87,6 +87,7 @@ KylinApp.service('modelsManager',function(ModelService,CubeService,$q,AccessServ
})
}
+
this.getModels = function(){
return _this.models;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/less/app.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index b6b3131..238e72e 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -788,3 +788,7 @@ input:-moz-placeholder {
.dropdown-menu{
z-index:9999;
}
+
+.panel-group .panel{
+ overflow: auto !important;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/partials/cubeDesigner/streamingConfig.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/streamingConfig.html b/webapp/app/partials/cubeDesigner/streamingConfig.html
index ed03229..c3afd26 100644
--- a/webapp/app/partials/cubeDesigner/streamingConfig.html
+++ b/webapp/app/partials/cubeDesigner/streamingConfig.html
@@ -18,22 +18,8 @@
<div ng-controller="streamingConfigCtrl">
<ng-form name="forms.cube_streaming_form" novalidate>
- <div class="form-group">
- <div class="row">
- <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Is this cube for streaming use?</b>
- </label>
- <div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'" >
- <toggle-switch ng-model="cubeState.isStreaming" on-label="YES" off-label="NO"> <toggle-switch>
- </div>
- <div class="col-xs-12 col-sm-6" ng-if="state.mode=='view'" >
- <span>{{(!!cubeState.isStreaming)?'YES':'NO'}}</span>
- </div>
- </div>
- </div>
-{{}}
- <div ng-if="cubeState.isStreaming">
+ <div>
<accordion>
<accordion-group is-open="state.isKfkSettingOpen" ng-init="state.isKfkSettingOpen=true">
<accordion-heading>
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/partials/tables/loadStreamingTable.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/loadStreamingTable.html b/webapp/app/partials/tables/loadStreamingTable.html
new file mode 100644
index 0000000..5e6116c
--- /dev/null
+++ b/webapp/app/partials/tables/loadStreamingTable.html
@@ -0,0 +1,89 @@
+
+<div class="modal-body streaming-source" style="height: 360px;">
+ <div class="col-xs-5">
+ <p class="text-info">
+ Need to input streaming source record here, will detect the source schema and create a table schema for
+ streaming.
+ </p>
+ <div class="has-error" ng-if="!table.sourceValid&&table.schemaChecked">
+ <small class="help-block">
+ Source json invalid.
+ </small>
+ </div>
+ <div style="padding:15px;" class="has-error">
+ <small class="help-block" ng-show="streaming.sourceSchema==''&&form.setStreamingSchema.$sbumitted">Please
+ input Streaming source record to generate schema.
+ </small>
+ </div>
+ <div style="margin-bottom: 20px;">
+ <span class="label label-info">JSON</span>
+ </div>
+ <div ng-model="streaming.sourceSchema" ui-ace="{
+ useWrapMode : true,
+ mode:'json',
+ onLoad: streamingOnLoad
+ }">
+
+ </div>
+ </div>
+ <div class="col-xs-1" style="margin-top:300px;text-align:center;">
+ <button type="button" class="btn btn-primary" ng-click="streamingOnChange()"><i
+ class="fa fa-angle-double-right fa-5" style="font-size:2em;"></i></button>
+ </div>
+ <div class="col-xs-6" ng-show="table.schemaChecked">
+ <ol class="text-info" style="margin-bottom: 30px;">
+ <li>Choose 'timestamp' type column for streaming table.</li>
+ <li>derived time dimensions are calculated from timestamp field to help analysis against different time granularities.</li>
+ </ol>
+ <form class="form-horizontal" name="form.setStreamingSchema" novalidate>
+ <div class="form-group required">
+ <label class="col-xs-4 control-label" style="text-align: left;">Table Name</label>
+
+ <div class="col-xs-8"
+ ng-class="{'has-error':form.setStreamingSchema.streamingObject.$invalid && (form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)}">
+ <input type="text" name="streamingObject" required="" ng-model="table.name" class="form-control"/>
+ <small class="help-block"
+ ng-show="form.setStreamingSchema.streamingObject.$error.required&&(form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)">
+ Table name is required.
+ </small>
+ </div>
+ </div>
+ </form>
+ <table class="table table-hover table-bordered">
+ <tr>
+ <th></th>
+ <th>Column</th>
+ <th>Column Type</th>
+ <th>Comment</th>
+ </tr>
+ <tr ng-repeat="column in columnList">
+ <td><label style="width:100%;cursor: pointer;" for="{{column.name}}"><input style="width:1em;height:1em;"
+ type="checkbox"
+ id="{{column.name}}"
+ ng-model="column.checked"
+ ng-true-value="Y"
+ ng-false-value="N"/></label>
+ </td>
+ <td>{{column.name}}</td>
+ <td>
+ <select chosen ng-model="column.type"
+ ng-options="type as type for type in tableConfig.dataTypes"
+ data-placeholder="select a column type"
+ style="width: 120px !important;"
+ class="chosen-select">
+ </select>
+ </td>
+ <td>
+ <label ng-if="column.type=='timestamp'&&column.fromSource=='Y'" class="badge badge-info">timestamp</label>
+ <label ng-if="column.fromSource=='N'" class="badge badge-info">derived time dimension</label>
+ </td>
+ </tr>
+ </table>
+
+ <div class="has-error" ng-if="!rule.timestampColumnExist&&form.setStreamingSchema.$sbumitted">
+ <small class="help-block">
+ You should choose at least one 'timestamp' type column generated from source schema.
+ </small>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/890577da/webapp/app/partials/tables/table_load.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/table_load.html b/webapp/app/partials/tables/table_load.html
index 29a2e9a..1fd4751 100644
--- a/webapp/app/partials/tables/table_load.html
+++ b/webapp/app/partials/tables/table_load.html
@@ -33,98 +33,40 @@
</script>
<script type="text/ng-template" id="addStreamingSource.html">
- <div class="modal-header">
- <h2>Create Streaming Table Schema</h2>
- </div>
- <div class="modal-body streaming-source" style="height: 660px;">
- <div class="col-xs-5">
- <p class="text-info">
- Need to input streaming source record here, will detect the source schema and create a table schema for
- streaming.
- </p>
-
- <div style="padding:15px;" class="has-error">
- <small class="help-block" ng-show="streaming.sourceSchema==''&&form.setStreamingSchema.$sbumitted">Please
- input Streaming source record to generate schema.
- </small>
- </div>
- <div style="margin-bottom: 20px;">
- <span class="label label-info">JSON</span>
- </div>
- <div ng-model="streaming.sourceSchema" ui-ace="{
- useWrapMode : true,
- mode:'json',
- onLoad: streamingOnLoad
- }">
+ <div class="modal-header">
+ <div class="box-header">
+ <h3 class="box-title">Streaming Table And Cluster Info</h3>
+ <div class="box-tools pull-right">
+ <button type="button" class="btn btn-box-tool" ng-click="cancel()" data-widget="remove"><i class="fa fa-times"></i></button>
</div>
</div>
- <div class="col-xs-1" style="margin-top:300px;text-align:center;">
- <button type="button" class="btn btn-primary" ng-click="streamingOnChange()"><i
- class="fa fa-angle-double-right fa-5" style="font-size:2em;"></i></button>
- </div>
- <div class="col-xs-6" ng-show="table.sourceValid">
- <ol class="text-info" style="margin-bottom: 30px;">
- <li>Choose 'timestamp' type column for streaming table.</li>
- <li>derived time dimensions are calculated from timestamp field to help analysis against different time granularities.</li>
- </ol>
- <form class="form-horizontal" name="form.setStreamingSchema" novalidate>
- <div class="form-group required">
- <label class="col-xs-4 control-label" style="text-align: left;">Table Name</label>
+ </div>
- <div class="col-xs-8"
- ng-class="{'has-error':form.setStreamingSchema.streamingObject.$invalid && (form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)}">
- <input type="text" name="streamingObject" required="" ng-model="table.name" class="form-control"/>
- <small class="help-block"
- ng-show="form.setStreamingSchema.streamingObject.$error.required&&(form.setStreamingSchema.streamingObject.$dirty||form.setStreamingSchema.$sbumitted)">
- Table name is required.
- </small>
- </div>
+ <div class="modal-body streaming-source" style="height: 660px;overflow-y:auto;">
+ <div ng-show="steps.curStep==1" ng-include="'partials/tables/loadStreamingTable.html'"></div>
+ <div ng-show="steps.curStep==2" ng-include="'partials/cubeDesigner/streamingConfig.html'"></div>
+ </div>
+ <div class="modal-footer">
+ <div class="row">
+ <div class="col-xs-8">
+ <div>
</div>
- </form>
- <table class="table table-hover table-bordered">
- <tr>
- <th></th>
- <th>Column</th>
- <th>Column Type</th>
- <th>Comment</th>
- </tr>
- <tr ng-repeat="column in columnList">
- <td><label style="width:100%;cursor: pointer;" for="{{column.name}}"><input style="width:1em;height:1em;"
- type="checkbox"
- id="{{column.name}}"
- ng-model="column.checked"
- ng-true-value="Y"
- ng-false-value="N"/></label>
- </td>
- <td>{{column.name}}</td>
- <td>
- <select chosen ng-model="column.type"
- ng-options="type as type for type in tableConfig.dataTypes"
- data-placeholder="select a column type"
- style="width: 120px !important;"
- class="chosen-select">
- </select>
- </td>
- <td>
- <label ng-if="column.type=='timestamp'&&column.fromSource=='Y'" class="badge badge-info">timestamp</label>
- <label ng-if="column.fromSource=='N'" class="badge badge-info">derived time dimension</label>
- </td>
- </tr>
- </table>
-
- <div class="has-error" ng-if="rule.timestampColumnConflict">
- <small class="help-block">
- You should choose one, and only one 'timestamp' type column generated from source schema.
- </small>
+ </div>
+ <div class="col-xs-4">
+ <button class="btn btn-prev" ng-click="previewStep();" ng-show="steps.curStep==2">
+ <i class="ace-icon fa fa-arrow-left"></i>
+ Prev
+ </button>
+ <button id="nextButton" class="btn btn-success btn-next" ng-click="nextStep();" ng-show="steps.curStep==1">
+ Next
+ <i class="ace-icon fa fa-arrow-right icon-on-right"></i>
+ </button>
+ <button class="btn btn-primary" ng-click="syncStreamingSchema()" ng-show="steps.curStep==2">
+ Submit
+ </button>
</div>
</div>
</div>
- <div class="modal-footer">
- <button class="btn btn-primary" ng-click="syncStreamingSchema()" ng-disabled="form.setStreamingSchema.$invalid">
- Submit
- </button>
- <button class="btn btn-primary" ng-click="cancel()">Cancel</button>
- </div>
</script>
[44/50] [abbrv] kylin git commit: KYLIN-1378 add topN UI support
Posted by sh...@apache.org.
KYLIN-1378 add topN UI support
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bccdd4fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bccdd4fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bccdd4fd
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: bccdd4fda268942661eb7c265b573d6824104574
Parents: 7377eda
Author: Jason <ji...@163.com>
Authored: Fri Mar 11 18:24:49 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Mar 11 18:36:00 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 6 +-
webapp/app/js/controllers/cubeMeasures.js | 8 +++
webapp/app/js/controllers/streamingConfig.js | 8 +--
webapp/app/js/directives/directives.js | 25 +++++++-
webapp/app/js/model/cubeConfig.js | 11 +++-
webapp/app/partials/cubeDesigner/measures.html | 65 +++++++++++++++++----
6 files changed, 103 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 748019f..4e9fdea 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -104,7 +104,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
//add cube dimension column for specific measure
angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension,index){
- if(dimension.column){
+ if(dimension.column && dimension.derived == null){
me_columns.push(dimension.column);
}
});
@@ -484,7 +484,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var newGroup = CubeDescModel.createAggGroup();
newGroup.includes = newUniqAggregationItem;
for(var i=1;i<initJointGroups.length;i++){
- newGroup.select_rule.joint_dims[i-1] = initJointGroups[i];
+ if(initJointGroups[i].length>1){
+ newGroup.select_rule.joint_dims[i-1] = initJointGroups[i];
+ }
}
$scope.cubeMetaFrame.aggregation_groups.push(newGroup);
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index f4bfab7..1e81e72 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -107,6 +107,10 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
//map right return type for param
$scope.measureReturnTypeUpdate = function(){
+ if($scope.newMeasure.function.expression == 'TOP_N'){
+ return;
+ }
+
if($scope.newMeasure.function.expression == 'COUNT'){
$scope.newMeasure.function.parameter.type= 'constant';
}
@@ -163,6 +167,10 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var NextParameterModalCtrl = function ($scope, scope,para,$modalInstance,cubeConfig, CubeService, MessageService, $location, SweetAlert,ProjectModel, loadingRequest,ModelService) {
+ $scope.newmea={
+ "measure":scope.newMeasure
+ }
+
$scope.cubeConfig = cubeConfig;
$scope.cancel = function () {
$modalInstance.dismiss('cancel');
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/js/controllers/streamingConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/streamingConfig.js b/webapp/app/js/controllers/streamingConfig.js
index 4e916db..7ba912f 100644
--- a/webapp/app/js/controllers/streamingConfig.js
+++ b/webapp/app/js/controllers/streamingConfig.js
@@ -21,9 +21,11 @@
KylinApp.controller('streamingConfigCtrl', function ($scope,StreamingService, $q, $routeParams, $location, $window, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, modelConfig, ProjectModel, ModelService, MetaModel, modelsManager, cubesManager, TableModel, $animate,StreamingModel) {
$scope.tableModel = TableModel;
- $scope.streamingMeta = StreamingModel.createStreamingConfig();
- $scope.kafkaMeta = StreamingModel.createKafkaConfig();
+ if($scope.state.mode=='view') {
+ $scope.streamingMeta = StreamingModel.createStreamingConfig();
+ $scope.kafkaMeta = StreamingModel.createKafkaConfig();
+ }
$scope.addCluster = function () {
@@ -105,8 +107,6 @@ KylinApp.controller('streamingConfigCtrl', function ($scope,StreamingService, $q
})
}
-
-
});
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index a9cd956..37c599d 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -271,4 +271,27 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
};
}
};
- });
+ }).directive("topntree", function($compile) {
+ return {
+ restrict: "E",
+ transclude: true,
+ scope: {
+ nextpara: '='
+ },
+ template:
+ '<li class="parent_li">Value:<b>{{nextpara.value}}</b>, Type:<b>{{ nextpara.type }}</b></li>' +
+ '<li class="parent_li">Order By:<b>{{nextpara.next_parameter.value}}</b></li>',
+ compile: function(tElement, tAttr, transclude) {
+ var contents = tElement.contents().remove();
+ var compiledContents;
+ return function(scope, iElement, iAttr) {
+ if(!compiledContents) {
+ compiledContents = $compile(contents, transclude);
+ }
+ compiledContents(scope, function(clone, scope) {
+ iElement.append(clone);
+ });
+ };
+ }
+ };
+});
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index b85789c..7747ecb 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,10 +20,9 @@ KylinApp.constant('cubeConfig', {
//~ Define metadata & class
measureParamType: ['column', 'constant'],
- measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT'],
+ measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N"],
dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
cubeCapacities: ["SMALL", "MEDIUM", "LARGE"],
-// cubePartitionTypes : ['APPEND', 'UPDATE_INSERT'],
cubePartitionTypes: ['APPEND'],
joinTypes: [
{name: 'Left', value: 'left'},
@@ -50,6 +49,11 @@ KylinApp.constant('cubeConfig', {
{name: 'Error Rate < 1.22%', value: 'hllc16'},
{name: 'Precisely (Only for Integer Family column)', value: 'bitmap'}
],
+ topNTypes: [
+ {name: 'Top 10', value: "topn(10)"},
+ {name: 'Top 100', value: "topn(100)"},
+ {name: 'Top 1000', value: "topn(1000)"}
+ ],
dftSelections: {
measureExpression: 'SUM',
measureParamType: 'column',
@@ -57,7 +61,8 @@ KylinApp.constant('cubeConfig', {
distinctDataType: {name: 'Error Rate < 4.88%', value: 'hllc12'},
cubeCapacity: 'MEDIUM',
queryPriority: {name: 'NORMAL', value: 50},
- cubePartitionType: 'APPEND'
+ cubePartitionType: 'APPEND',
+ topN:{name: 'Top 100', value: "topn(100)"}
},
dictionaries: ["true", "false"],
http://git-wip-us.apache.org/repos/asf/kylin/blob/bccdd4fd/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 548cb13..b58f200 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -43,7 +43,8 @@
<td>
<div class="paraTree">
<ul>
- <parametertree ng-if="measure.function.parameter!=null" nextpara="measure.function.parameter"></parametertree>
+ <parametertree ng-if="measure.function.parameter!=null && measure.function.expression!=='TOP_N'" nextpara="measure.function.parameter"></parametertree>
+ <topntree ng-if="measure.function.parameter!=null && measure.function.expression=='TOP_N'" nextpara="measure.function.parameter"></topntree>
</ul>
</div>
<!--<span ng-if="measure.function.parameter.next_parameter!=null">{{measure.function.parameter.next_parameter |json}}</span>-->
@@ -138,7 +139,7 @@
ng-init="newMeasure.function.parameter.value = 1"><b> 1</b></span>
<!--!COUNT_DISTINCT-->
<select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression!=='COUNT_DISTINCT'"
+ ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression!=='COUNT_DISTINCT'&& newMeasure.function.expression!=='TOP_N'"
ng-model="newMeasure.function.parameter.value"
ng-change="measureReturnTypeUpdate();"
ng-options="column as column for column in getCommonMetricColumns()" >
@@ -146,7 +147,7 @@
</select>
<!--COUNT_DISTINCT-->
<select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression=='COUNT_DISTINCT'"
+ ng-if="newMeasure.function.parameter.type == 'column' && (newMeasure.function.expression=='COUNT_DISTINCT' || newMeasure.function.expression=='TOP_N')"
ng-model="newMeasure.function.parameter.value"
ng-change="measureReturnTypeUpdate();"
ng-options="column as column for column in getMetricColumns()" >
@@ -167,8 +168,15 @@
ng-options="ddt.value as ddt.name for ddt in cubeConfig.distinctDataTypes">
<option value=""></option>
</select>
+ <select class="form-control"
+ ng-if="newMeasure.function.expression == 'TOP_N'"
+ ng-init="newMeasure.function.returntype = (!!newMeasure.function.returntype)?newMeasure.function.returntype:cubeConfig.dftSelections.topN.value"
+ chosen ng-model="newMeasure.function.returntype" required
+ ng-options="ddt.value as ddt.name for ddt in cubeConfig.topNTypes">
+ <option value=""></option>
+ </select>
<span class="font-color-default"
- ng-if="newMeasure.function.expression != 'COUNT_DISTINCT'"
+ ng-if="newMeasure.function.expression != 'COUNT_DISTINCT' && newMeasure.function.expression != 'TOP_N'"
><b> {{newMeasure.function.returntype | uppercase}}</b>
</span>
</div>
@@ -179,7 +187,7 @@
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b></b></label>
<div class="col-xs-12 col-sm-6">
- <table class="table table-hover table-bordered list" ng-if="nextParameters.length">
+ <table class="table table-hover table-bordered list" ng-if="nextParameters.length" ng-show="newMeasure.function.expression != 'TOP_N'">
<tr>
<th>Type</th>
<th>Value</th>
@@ -199,9 +207,30 @@
</tr>
</table>
- <button class="btn btn-sm btn-info" ng-click="addNextParameter()"
+
+ <table class="table table-hover table-bordered list" ng-if="nextParameters.length" ng-show="newMeasure.function.expression == 'TOP_N'">
+ <tr ng-repeat="n_parameter in nextParameters track by $index">
+ <td><b>Order By Column</b></td>
+ <td>{{n_parameter.value}}</td>
+ <td>
+ <button class="btn btn-xs btn-info" ng-click="editNextParameter(n_parameter)">
+ <i class="fa fa-pencil"></i>
+ </button>
+ <button class="btn btn-xs btn-info" ng-click="removeParameter(nextParameters, $index)"><i class="fa fa-minus"></i>
+ </button>
+
+ </td>
+
+ </tr>
+ </table>
+
+
+ <button class="btn btn-sm btn-info" ng-click="addNextParameter()" ng-show="newMeasure.function.expression != 'TOP_N'"
ng-show="state.mode=='edit'"><i class="fa fa-plus">Parameter</i>
</button>
+ <button class="btn btn-sm btn-info" ng-click="addNextParameter()" ng-show="newMeasure.function.expression == 'TOP_N' && nextParameters.length==0"
+ ng-show="state.mode=='edit'"><i class="fa fa-plus"> Order by Column</i>
+ </button>
</div>
</div>
</div>
@@ -242,7 +271,8 @@
<script type="text/ng-template" id="nextParameter.html">
<div class="modal-header">
- <h4 tooltip="submit">Add Parameter</h4>
+ <h4 tooltip="submit" ng-if="newmea.measure&&newmea.measure.function.expression !== 'TOP_N'">Add Parameter</h4>
+ <h4 tooltip="submit" ng-if="newmea.measure&&newmea.measure.function.expression == 'TOP_N'">Select Order By Column</h4>
</div>
<div class="modal-body" style="background-color: white">
@@ -251,7 +281,7 @@
<div class="col-md-8">
<div class="row">
<div class="form-group">
- <div class="row">
+ <div class="row" ng-if="newmea.measure&&newmea.measure.function.expression !== 'TOP_N'">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Param Type</b></label>
<div class="col-xs-12 col-sm-6">
<select class="form-control"
@@ -272,8 +302,9 @@
<div class="col-md-8">
<div class="row">
<div class="form-group">
- <div class="row">
- <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Param Value</b></label>
+
+ <div ng-if="newmea.measure&&newmea.measure.function.expression !== 'TOP_N'" class="row">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Param Value</b></label>
<!--COUNT_DISTINCT-->
<div class="col-xs-12 col-sm-6">
<select class="form-control" chosen ng-if="nextPara.type !== 'constant'"
@@ -285,8 +316,22 @@
ng-if="nextPara.type == 'constant'"
ng-init="nextPara.value = 1"><b> 1</b></span>
</div>
+ </div>
+
+ <div ng-if="newmea.measure&&newmea.measure.function.expression == 'TOP_N'" ng-init="nextPara.type='column'" class="row">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Order By Column</b></label>
+ <!--COUNT_DISTINCT-->
+ <div class="col-xs-12 col-sm-6">
+ <select class="form-control" chosen
+ ng-model="nextPara.value"
+ ng-options="column as column for column in getMetricColumns()" >
+ <option value=""></option>
+ </select>
</div>
</div>
+
+
+ </div>
</div>
</div>
</div>
[16/50] [abbrv] kylin git commit: KYLIN-1417 Change to use TreeMap to
allow null as value
Posted by sh...@apache.org.
KYLIN-1417 Change to use TreeMap to allow null as value
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1a574b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1a574b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1a574b8
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: d1a574b80ef35ebdae69718998a2dfcaeafc3cbc
Parents: 50aab0b
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 10:29:47 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Mar 8 12:21:40 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/source/kafka/TimedJsonStreamParser.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1a574b8/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index 0907623..e3075d5 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -35,11 +35,7 @@
package org.apache.kylin.source.kafka;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import kafka.message.MessageAndOffset;
@@ -102,7 +98,9 @@ public final class TimedJsonStreamParser extends StreamingParser {
@Override
public StreamingMessage parse(MessageAndOffset messageAndOffset) {
try {
- Map<String, String> root = mapper.readValue(new ByteBufferBackedInputStream(messageAndOffset.message().payload()), mapType);
+ Map<String, String> message = mapper.readValue(new ByteBufferBackedInputStream(messageAndOffset.message().payload()), mapType);
+ Map<String, String> root = new TreeMap<String, String>(String.CASE_INSENSITIVE_ORDER);
+ root.putAll(message);
String tsStr = root.get(tsColName);
//Preconditions.checkArgument(!StringUtils.isEmpty(tsStr), "Timestamp field " + tsColName + //
//" cannot be null, the message offset is " + messageAndOffset.getOffset() + " content is " + new String(messageAndOffset.getRawData()));
[39/50] [abbrv] kylin git commit: KYLIN-1286 fix java license files
Posted by sh...@apache.org.
KYLIN-1286 fix java license files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d0e096a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d0e096a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d0e096a7
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: d0e096a77f13e235fec5ee576e1f99169cb74fce
Parents: aa49ece
Author: Li Yang <li...@apache.org>
Authored: Fri Mar 11 16:04:33 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Mar 11 16:04:33 2016 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/RawResource.java | 18 +++
.../kylin/common/util/Log4jConfigurer.java | 18 +++
.../org/apache/kylin/common/util/RangeUtil.java | 18 +++
.../org/apache/kylin/common/util/SortUtil.java | 18 +++
.../kylin/common/util/StreamingMessage.java | 18 +++
.../org/apache/kylin/common/util/TimeUtil.java | 18 +++
.../kylin/common/util/ITMockHTableTest.java | 158 -------------------
.../kylin/common/util/IdentityUtilTest.java | 18 +++
.../org/apache/kylin/common/util/RangeTest.java | 18 +++
.../apache/kylin/common/util/RangeUtilTest.java | 18 +++
.../apache/kylin/common/util/TimeUtilTest.java | 18 +++
.../org/apache/kylin/dict/DictCodeSystem.java | 18 +++
.../apache/kylin/dict/DictionarySerializer.java | 18 +++
.../dict/DistinctColumnValuesProvider.java | 18 +++
.../apache/kylin/dict/TimeStrDictionary.java | 18 +++
.../apache/kylin/dict/lookup/SnapshotCLI.java | 18 +++
.../kylin/dict/TimeStrDictionaryTests.java | 18 +++
.../kylin/measure/bitmap/BitmapMeasureType.java | 18 +++
.../metadata/datatype/DateTimeSerializer.java | 18 +++
.../metadata/datatype/StringSerializer.java | 18 +++
.../realization/IRealizationSegment.java | 18 +++
.../metadata/realization/SQLDigestUtil.java | 18 +++
.../metadata/realization/StreamSQLDigest.java | 18 +++
.../metadata/tuple/CompoundTupleIterator.java | 18 +++
.../kylin/metadata/tuple/IEvaluatableTuple.java | 18 +++
.../metadata/tuple/SimpleTupleIterator.java | 18 +++
.../kylin/metadata/tuple/TeeTupleIterator.java | 18 +++
.../metadata/tuple/TeeTupleItrListener.java | 18 +++
.../topn/TopNCounterSerializerTest.java | 18 +++
.../datatype/BigDecimalSerializerTest.java | 18 +++
30 files changed, 522 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/RawResource.java b/core-common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
index c0b5b21..6c155a2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.persistence;
import java.io.InputStream;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java b/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
index 6f887b7..a8540af 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Enumeration;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
index 33dfb83..c071f98 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
index add7b67..1ea9d97 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
index 2c150ff..f327db2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index 17868a6..d3d4283 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Calendar;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java b/core-common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
deleted file mode 100644
index 74caa4a..0000000
--- a/core-common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-//package org.apache.kylin.common.util;
-//
-//import com.google.common.collect.Iterables;
-//import com.google.common.collect.Lists;
-//import org.apache.hadoop.hbase.client.*;
-//import org.apache.hadoop.hbase.filter.CompareFilter;
-//import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-//import org.apache.kylin.common.KylinConfig;
-//import org.apache.kylin.common.persistence.HBaseConnection;
-//import org.junit.AfterClass;
-//import org.junit.BeforeClass;
-//import org.junit.Test;
-//import org.slf4j.Logger;
-//import org.slf4j.LoggerFactory;
-//
-//import java.io.IOException;
-//
-//import static org.hamcrest.MatcherAssert.assertThat;
-//import static org.hamcrest.Matchers.is;
-//
-///**
-// * MockHTable Test.
-// *
-// * compare to real HTable.
-// */
-//
-//public class ITMockHTableTest extends HBaseMetadataTestCase {
-// private final Logger logger = LoggerFactory.getLogger(this.getClass());
-//
-// private static HTableInterface mock;
-// private static HTableInterface real;
-// private static String emailCF = "EMAIL";
-// private static String addressCF = "ADDRESS";
-// private static String emailCQ = "email";
-// private static String addressCQ = "address";
-// private static String tableName = "MockHTable-TEST";
-//
-// @BeforeClass
-// public static void setUp() throws Exception {
-//
-// staticCreateTestMetadata();
-//
-// HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
-// HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
-// real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
-// mock = new MockHTable(tableName, emailCF, addressCF);
-//
-// setupDefaultData();
-// }
-//
-// @AfterClass
-// public static void tearDown() throws Exception {
-// HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
-// staticCleanupTestMetadata();
-// }
-//
-// private static void setupDefaultData() throws IOException, InterruptedException {
-// Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
-// Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
-// Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
-// mutateAll(john, jane, me);
-// }
-//
-// private static Put createPutForPerson(String name, String email, String address) {
-// Put person = new Put(name.getBytes());
-// person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
-// person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
-// return person;
-// }
-//
-// private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
-// real.batch(Lists.newArrayList(rows));
-// mock.batch(Lists.newArrayList(rows));
-// }
-//
-// @Test
-// public void test_get() throws Exception {
-// Get get = new Get("John Doe".getBytes());
-//
-// Result realResult = real.get(get);
-// Result mockResult = mock.get(get);
-//
-// assertThat(realResult.isEmpty(), is(false));
-// assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
-// }
-//
-// @Test
-// public void test_get_with_filter() throws Exception {
-// Get get = new Get("John Doe".getBytes());
-// get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
-//
-// ));
-//
-// Result realResult = real.get(get);
-// Result mockResult = mock.get(get);
-//
-// assertThat(realResult.isEmpty(), is(true));
-// assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
-// }
-//
-// @Test
-// public void test_exists() throws IOException {
-// Get get = new Get("John Doe".getBytes());
-// boolean realResult = real.exists(get);
-// boolean mockResult = mock.exists(get);
-//
-// assertThat(realResult, is(true));
-// assertThat(realResult, is(mockResult));
-// }
-//
-// @Test
-// public void test_exists_include_not_exist_column() throws IOException {
-// Get get = new Get("John Doe".getBytes());
-// get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
-// get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
-// boolean realResult = real.exists(get);
-// boolean mockResult = mock.exists(get);
-//
-// assertThat(realResult, is(true));
-// assertThat(realResult, is(mockResult));
-// }
-//
-// @Test
-// public void test_exists_with_only_not_exist_column() throws IOException {
-// Get get = new Get("John Doe".getBytes());
-// get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
-// boolean realResult = real.exists(get);
-// boolean mockResult = mock.exists(get);
-//
-// assertThat(realResult, is(false));
-// assertThat(realResult, is(mockResult));
-// }
-//
-// @Test
-// public void test_scan_with_filter() throws Exception {
-// Scan scan = new Scan();
-// scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
-//
-// ResultScanner realResult = real.getScanner(scan);
-// ResultScanner mockResult = mock.getScanner(scan);
-//
-// logger.debug("mock : {}", mockResult);
-//
-// assertThat(Iterables.size(realResult), is(2));
-// assertThat(Iterables.size(mockResult), is(2));
-// }
-//
-// @Test
-// public void test_scan_for_pre_match() throws Exception {
-// Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
-//
-// ResultScanner realResult = real.getScanner(scan);
-// ResultScanner mockResult = mock.getScanner(scan);
-//
-// assertThat(Iterables.size(realResult), is(2));
-// assertThat(Iterables.size(mockResult), is(2));
-// }
-//}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
index f3fa078..c86e577 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/test/java/org/apache/kylin/common/util/RangeTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/RangeTest.java b/core-common/src/test/java/org/apache/kylin/common/util/RangeTest.java
index 0b8730c..f39c06b 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/RangeTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/RangeTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
index 70f8767..1601fe7 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index 3fdf6aa..15f54f9 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.common.util;
import java.text.ParseException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
index b0326c1..158ce86 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
index 6b47868..72dc995 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import java.io.ByteArrayInputStream;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java
index 66511be..c91cbbb 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DistinctColumnValuesProvider.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index 65c6c05..3c96d08 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import java.io.DataInput;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
index 149badc..45fd56b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict.lookup;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
index 1e381d4..a62bc84 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import org.apache.kylin.common.util.DateFormat;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index df8e765..b59be4e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.measure.bitmap;
import org.apache.kylin.common.util.Dictionary;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
index fb1c65b..b7156b4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.datatype;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
index eef2868..235808f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.datatype;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationSegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationSegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationSegment.java
index 3ac82a0..6e3932d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationSegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationSegment.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.realization;
import org.apache.kylin.metadata.model.IBuildable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index e52bf3b..a70a17e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.realization;
import org.apache.kylin.common.util.DateFormat;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
index 1420470..345350e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.realization;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
index 02af3ca..fff910b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.tuple;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
index d5a5061..66c165f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.tuple;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
index 92006e1..6b434f2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.tuple;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
index c12f18a..e67443f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.tuple;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
index 750769a..b5fca16 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.tuple;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
index 5b50241..1da3624 100644
--- a/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.aggregation.topn;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d0e096a7/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
index dcdc3d4..1c83af4 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.metadata.datatype;
import static org.junit.Assert.assertEquals;
[22/50] [abbrv] kylin git commit: minor,
update find hive dependency script to avoid soft links of jars
Posted by sh...@apache.org.
minor, update find hive dependency script to avoid soft links of jars
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26233f79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26233f79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26233f79
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 26233f79bf7fdc3fa1de9ce82e36d0baf47dcd20
Parents: f73abf6
Author: lidongsjtu <li...@apache.org>
Authored: Wed Mar 9 16:14:11 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Mar 9 16:16:37 2016 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/26233f79/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 87f18dd..1650ec0 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -41,7 +41,7 @@ fi
for data in ${arr[@]}
do
- result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*jar'`
+ result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*.jar'`
if [ $result ]
then
hive_exec_path=$data
@@ -82,7 +82,7 @@ else
hcatalog_home=${HCAT_HOME}
fi
-hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*.jar" 2>&1 | grep -m 1 -v 'Permission denied'`
if [ -z "$hcatalog" ]
then
[26/50] [abbrv] kylin git commit: bug fix,
new aggr group does not support hierarchy well
Posted by sh...@apache.org.
bug fix, new aggr group does not support hierarchy well
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7db093f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7db093f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7db093f
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: b7db093fbefd9fac1e2193e8e06ce64db2bd0ce0
Parents: 4e48f2a
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 10 18:58:21 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 10 18:58:37 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/cuboid/Cuboid.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7db093f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 2cd96a6..c4e373b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -143,9 +143,18 @@ public class Cuboid implements Comparable<Cuboid> {
long fullMask = hierarchyMask.fullMask;
long intersect = cuboidID & fullMask;
if (intersect != 0 && intersect != fullMask) {
- long lsb = Long.lowestOneBit(intersect);
- long fillMask = fullMask & ~(lsb - 1);
- cuboidID |= fillMask;
+
+ boolean startToFill = false;
+ for (int i = hierarchyMask.dims.length - 1; i >= 0; i--) {
+ if (startToFill) {
+ cuboidID |= hierarchyMask.dims[i];
+ } else {
+ if ((cuboidID & hierarchyMask.dims[i]) != 0) {
+ startToFill = true;
+ cuboidID |= hierarchyMask.dims[i];
+ }
+ }
+ }
}
}
[24/50] [abbrv] kylin git commit: KYLIN-1483 Command tool to
visualize all cuboids in a cube/segment
Posted by sh...@apache.org.
KYLIN-1483 Command tool to visualize all cuboids in a cube/segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4be5461
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4be5461
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4be5461
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: c4be5461f070b073c12388d0ad9d8d9d3b7519f0
Parents: ed57c3b
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 10 14:21:57 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 10 14:21:57 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/cuboid/Cuboid.java | 39 +++++++++++---------
.../kylin/engine/mr/common/CubeStatsReader.java | 7 ++--
2 files changed, 25 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c4be5461/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 513513c..2cd96a6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -18,14 +18,11 @@
package org.apache.kylin.cube.cuboid;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.AggregationGroup;
@@ -34,10 +31,13 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
public class Cuboid implements Comparable<Cuboid> {
@@ -222,7 +222,8 @@ public class Cuboid implements Comparable<Cuboid> {
return true;
}
- hier: for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
+ hier:
+ for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
long result = cuboidID & hierarchyMasks.fullMask;
if (result > 0) {
// if match one of the hierarchy constrains, return true;
@@ -385,11 +386,13 @@ public class Cuboid implements Comparable<Cuboid> {
public static String getDisplayName(long cuboidID, int dimensionCount) {
StringBuilder sb = new StringBuilder();
- String temp = Long.toString(cuboidID);
- for (int i = 0; i < dimensionCount - temp.length(); i++) {
- sb.append("0");
+ for (int i = 0; i < dimensionCount; ++i) {
+ if ((cuboidID & (1L << i)) == 0) {
+ sb.append('0');
+ } else {
+ sb.append('1');
+ }
}
- sb.append(temp);
- return sb.toString();
+ return StringUtils.reverse(sb.toString());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c4be5461/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index e842c01..57e93c3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -241,7 +241,7 @@ public class CubeStatsReader {
CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
int dimensionCount = Long.bitCount(baseCuboid);
- printCuboidInfoTree(0L, baseCuboid, scheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
+ printCuboidInfoTree(-1L, baseCuboid, scheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
}
private static void printCuboidInfoTree(long parent, long cuboidID, final CuboidScheduler scheduler, Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, int dimensionCount, int depth, PrintWriter out) {
@@ -251,7 +251,7 @@ public class CubeStatsReader {
Collections.sort(children);
for (Long child : children) {
- printCuboidInfoTree(parent, child, scheduler, cuboidRows, cuboidSizes, dimensionCount, depth + 1, out);
+ printCuboidInfoTree(cuboidID, child, scheduler, cuboidRows, cuboidSizes, dimensionCount, depth + 1, out);
}
}
@@ -268,7 +268,7 @@ public class CubeStatsReader {
sb.append(", est row: ").append(rowCount).append(", est MB: ").append(formatDouble(size));
if (parent != -1) {
- sb.append(", shrink: ").append(formatDouble(1.0 * cuboidRows.get(cuboidID) / cuboidRows.get(parent))).append("%");
+ sb.append(", shrink: ").append(formatDouble(100.0 * cuboidRows.get(cuboidID) / cuboidRows.get(parent))).append("%");
}
out.println(sb.toString());
@@ -279,6 +279,7 @@ public class CubeStatsReader {
}
public static void main(String[] args) throws IOException {
+ System.out.println("CubeStatsReader is used to read cube statistic saved in metadata store");
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeInstance cube = CubeManager.getInstance(config).getCube(args[0]);
List<CubeSegment> segments = cube.getSegments(SegmentStatusEnum.READY);