You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/12 11:20:49 UTC
[01/35] kylin git commit: KYLIN-1926 check pk,
fk type when edit model join relation [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/1.5.x-HBase1.x b70684e66 -> 6d69da921 (forced update)
KYLIN-1926 check pk,fk type when edit model join relation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f12a5e77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f12a5e77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f12a5e77
Branch: refs/heads/1.5.x-HBase1.x
Commit: f12a5e77e81d88a89aa700dd2dccba7f2e0814ae
Parents: 4ede67e
Author: Jason <ji...@163.com>
Authored: Wed Sep 7 11:39:08 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 11:39:28 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/data_model.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f12a5e77/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 662d0d9..ace977f 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -173,7 +173,7 @@
</button>
</div>
<div class="space-4"></div>
- <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}],{{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}].</b></small>
+ <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}], {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}]</b></small>
</div>
</div>
</div>
[08/35] kylin git commit: minor, add one more test case
Posted by li...@apache.org.
minor, add one more test case
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01d033f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01d033f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01d033f7
Branch: refs/heads/1.5.x-HBase1.x
Commit: 01d033f76a058ca0cf7ef81728b7de6204115167
Parents: ddeb745
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Sep 8 11:18:52 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 8 11:18:52 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeCapabilityChecker.java | 7 ++++++-
kylin-it/src/test/resources/query/sql_like/query18.sql | 13 +++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/01d033f7/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 79d1e3b..e8c96b4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -31,6 +31,7 @@ import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -146,7 +147,11 @@ public class CubeCapabilityChecker {
}
// calcite can do aggregation from columns on-the-fly
- List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
+ ParameterDesc parameterDesc = functionDesc.getParameter();
+ if (parameterDesc == null) {
+ continue;
+ }
+ List<TblColRef> neededCols = parameterDesc.getColRefs();
if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
it.remove();
http://git-wip-us.apache.org/repos/asf/kylin/blob/01d033f7/kylin-it/src/test/resources/query/sql_like/query18.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query18.sql b/kylin-it/src/test/resources/query/sql_like/query18.sql
new file mode 100644
index 0000000..8ef6ad4
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query18.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where upper(USER_DEFINED_FIELD3) like '%VID%'
[04/35] kylin git commit: KYLIN-1698-INT
Posted by li...@apache.org.
KYLIN-1698-INT
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/413bc9f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/413bc9f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/413bc9f2
Branch: refs/heads/1.5.x-HBase1.x
Commit: 413bc9f2d6c73c3c4822ba3ce0ab9da405cd5b9c
Parents: e3a1767
Author: chenzhx <34...@qq.com>
Authored: Wed Sep 7 13:48:21 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 15:49:31 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelEdit.js | 7 +++++--
webapp/app/partials/modelDesigner/conditions_settings.html | 4 ++--
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/413bc9f2/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 2532fb4..65e17fa 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint";
+ return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint"||column.datatype==="int";
});
return columns;
};
@@ -71,12 +71,15 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.isBigInt=false;
$scope.partitionChange = function (dateColumn) {
+ if(dateColumn==null) {
+ return;
+ }
var column = _.filter($scope.getColumnsByTable($scope.modelsManager.selectedModel.fact_table),function(_column){
var columnName=$scope.modelsManager.selectedModel.fact_table+"."+_column.name;
if(dateColumn==columnName)
return _column;
});
- if(column[0].datatype==="bigint"){
+ if(column[0].datatype==="bigint"||column[0].datatype==="int"){
$scope.isBigInt=true;
$scope.modelsManager.selectedModel.partition_desc.partition_date_format=null;;
$scope.partitionColumn.hasSeparateTimeColumn=false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/413bc9f2/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 693241c..f0390e5 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -67,7 +67,7 @@
<select style="width: 100%" chosen
ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-if="state.mode=='edit'" ng-disable="isBigInt(modelsManager.selectedModel.partition_desc.partition_date_column)"
+ ng-if="state.mode=='edit'"
data-placement=""
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
<option value="">--Select Date Format--</option>
@@ -164,7 +164,7 @@
<script type="text/ng-template" id="partitionTip.html">
<ol>
<li>Partition date column not required,leave as default if cube always need full build</Li>
- <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR,BigInt, etc.)</li>
+ <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR, Int, BigInt, etc.)</li>
</ol>
</script>
[05/35] kylin git commit: KYLIN-1998 release job engine lock at
shutdown
Posted by li...@apache.org.
KYLIN-1998 release job engine lock at shutdown
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/97778788
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/97778788
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/97778788
Branch: refs/heads/1.5.x-HBase1.x
Commit: 97778788a0fea2bb4db4254264d42c58bdb8f7cc
Parents: 413bc9f
Author: Li Yang <li...@apache.org>
Authored: Wed Sep 7 17:37:57 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Sep 7 17:38:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/97778788/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 2dc1ab5..1ea3be0 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -49,6 +49,7 @@ import com.google.common.collect.Maps;
*/
public class DefaultScheduler implements Scheduler<AbstractExecutable>, ConnectionStateListener {
+ private JobLock jobLock;
private ExecutableManager executableManager;
private FetcherRunner fetcher;
private ScheduledExecutorService fetcherPool;
@@ -181,6 +182,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, final JobLock jobLock) throws SchedulerException {
+ this.jobLock = jobLock;
+
String serverMode = jobEngineConfig.getConfig().getServerMode();
if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) {
logger.info("server mode: " + serverMode + ", no need to run job scheduler");
@@ -216,6 +219,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
@Override
public void shutdown() throws SchedulerException {
+ logger.info("Shutingdown Job Engine ....");
+ jobLock.unlock();
fetcherPool.shutdown();
jobPool.shutdown();
}
[12/35] kylin git commit: minor, add diagnosis to smoke test
Posted by li...@apache.org.
minor, add diagnosis to smoke test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf261148
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf261148
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf261148
Branch: refs/heads/1.5.x-HBase1.x
Commit: bf26114866d9b4f8dcb5e13c48acdbf6c8e308e9
Parents: be7751b
Author: lidongsjtu <li...@apache.org>
Authored: Fri Sep 9 12:37:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Sep 9 12:37:05 2016 +0800
----------------------------------------------------------------------
build/smoke-test/smoke-test.sh | 1 +
build/smoke-test/testDiag.py | 44 +++++++++++++++++++++++++++++++++++++
2 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf261148/build/smoke-test/smoke-test.sh
----------------------------------------------------------------------
diff --git a/build/smoke-test/smoke-test.sh b/build/smoke-test/smoke-test.sh
index 06d930a..c21bd6d 100755
--- a/build/smoke-test/smoke-test.sh
+++ b/build/smoke-test/smoke-test.sh
@@ -67,6 +67,7 @@ sleep 3m
cd $dir/smoke-test
python testBuildCube.py || { exit 1; }
python testQuery.py || { exit 1; }
+python testDiag.py || { exit 1; }
cd -
# Tear down stage
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf261148/build/smoke-test/testDiag.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
new file mode 100644
index 0000000..cc932da
--- /dev/null
+++ b/build/smoke-test/testDiag.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is python unittest used in smoke-test.sh, aim to testing query via rest APIs.
+
+import unittest
+import requests
+
+class testDiag(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def testDiag(self):
+ url = "http://sandbox:7070/kylin/api/diag/project/learn_kylin/download"
+ headers = {
+ 'content-type': "application/json",
+ 'authorization': "Basic QURNSU46S1lMSU4=",
+ 'cache-control': "no-cache"
+ }
+
+ response = requests.get(url, headers = headers)
+ self.assertEqual(response.status_code, 200, 'Diagnosis failed.')
+
+
+if __name__ == '__main__':
+ print 'Test Diagnogis for Kylin sample.'
+ unittest.main()
[19/35] kylin git commit: KYLIN-1922 fix CI
Posted by li...@apache.org.
KYLIN-1922 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/466cf1af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/466cf1af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/466cf1af
Branch: refs/heads/1.5.x-HBase1.x
Commit: 466cf1afb19ee1acb4f99ff72ac61e7689617957
Parents: 618cf28
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 22:27:17 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 22:27:17 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITKylinQueryTest.java | 53 ++++++++------------
1 file changed, 21 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/466cf1af/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 0efea64..5f6af7a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
@@ -40,12 +41,9 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -122,41 +120,32 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
- thrown.expect(SQLException.class);
-
- //should not break at table duplicate check, should fail at model duplicate check
- thrown.expectCause(new BaseMatcher<Throwable>() {
- @Override
- public boolean matches(Object item) {
- if (item instanceof GTScanSelfTerminatedException) {
- return true;
- }
- return false;
- }
-
- @Override
- public void describeTo(Description description) {
- }
- });
+ try {
- Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
- BackdoorToggles.setToggles(toggles);
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ BackdoorToggles.setToggles(toggles);
- KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
- //these two cubes has RAW measure, will disturb limit push down
- RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+ } catch (SQLException e) {
+ if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
+ throw new RuntimeException();
+ }
+ } finally {
- //these two cubes has RAW measure, will disturb limit push down
- RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
- KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
- BackdoorToggles.cleanToggles();
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
+ BackdoorToggles.cleanToggles();
+ }
}
//don't try to ignore this test, try to clean your "temp" folder
[03/35] kylin git commit: KYLIN-1698-BIGINT
Posted by li...@apache.org.
KYLIN-1698-BIGINT
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3a1767f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3a1767f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3a1767f
Branch: refs/heads/1.5.x-HBase1.x
Commit: e3a1767f07dcb7365480d0fa495083d8bfdd0344
Parents: f12a5e7
Author: chenzhx <34...@qq.com>
Authored: Tue Sep 6 17:20:15 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 15:48:50 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelEdit.js | 25 +++++++++++++++++++-
.../modelDesigner/conditions_settings.html | 10 ++++----
2 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a1767f/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index a916aad..2532fb4 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar");
+ return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint";
});
return columns;
};
@@ -69,6 +69,26 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
return type;
};
+ $scope.isBigInt=false;
+ $scope.partitionChange = function (dateColumn) {
+ var column = _.filter($scope.getColumnsByTable($scope.modelsManager.selectedModel.fact_table),function(_column){
+ var columnName=$scope.modelsManager.selectedModel.fact_table+"."+_column.name;
+ if(dateColumn==columnName)
+ return _column;
+ });
+ if(column[0].datatype==="bigint"){
+ $scope.isBigInt=true;
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_format=null;;
+ $scope.partitionColumn.hasSeparateTimeColumn=false;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
+ }
+ else{
+ $scope.isBigInt=false;
+ }
+
+ }
+
// ~ Define data
$scope.state = {
"modelSchema": "",
@@ -87,6 +107,9 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
ModelDescService.query({model_name: modelName}, function (model) {
if (model) {
modelsManager.selectedModel = model;
+ if(!$scope.modelsManager.selectedModel.partition_desc.partition_data_format){
+ $scope.isBigInt = true;
+ }
if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
$scope.partitionColumn.hasSeparateTimeColumn = true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a1767f/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index a4e49a9..693241c 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -46,7 +46,7 @@
<select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
- ng-if="state.mode=='edit'"
+ ng-if="state.mode=='edit'" ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonColumns(modelsManager.selectedModel.fact_table)" >
<option value="">--Select Partition Column--</option>
@@ -60,14 +60,14 @@
</div>
<!--Date Format-->
- <div class="form-group">
+ <div class="form-group" ng-if="isBigInt==false">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
<div class="col-xs-12 col-sm-6">
<select style="width: 100%" chosen
ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-if="state.mode=='edit'"
+ ng-if="state.mode=='edit'" ng-disable="isBigInt(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
<option value="">--Select Date Format--</option>
@@ -78,7 +78,7 @@
</div>
<!--Date Format-->
- <div class="form-group middle-popover">
+ <div class="form-group middle-popover" ng-if="isBigInt==false">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has a separate "time of the day" column ?</b> <i kylinpopover placement="right" title="Separate Time Column" template="separateTimeColumnTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6">
@@ -164,7 +164,7 @@
<script type="text/ng-template" id="partitionTip.html">
<ol>
<li>Partition date column not required,leave as default if cube always need full build</Li>
- <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR, etc.)</li>
+ <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR,BigInt, etc.)</li>
</ol>
</script>
[29/35] kylin git commit: KYLIN-1983 add license header
Posted by li...@apache.org.
KYLIN-1983 add license header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5c85017
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5c85017
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5c85017
Branch: refs/heads/1.5.x-HBase1.x
Commit: c5c85017c135f03ac72ebbc9d1bb51f796eb0551
Parents: b941f11
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 09:58:48 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamDataLoader.java | 19 ++++++++++++++++++-
.../model/validation/rule/FunctionRuleTest.java | 18 ++++++++++++++++++
.../optrule/AggregateMultipleExpandRule.java | 18 ++++++++++++++++++
.../optrule/AggregateProjectReduceRule.java | 18 ++++++++++++++++++
.../PasswordPlaceHolderConfigurerTest.java | 18 ++++++++++++++++++
tomcat-ext/pom.xml | 18 ++++++++++++++++++
6 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
index 50fc883..2f7d54d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
@@ -1,6 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.job.streaming;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
index 48e01e3..e041080 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.cube.model.validation.rule;
import org.apache.kylin.common.KylinConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
index eb7a03d..03a0674 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.query.optrule;
import com.google.common.base.Predicate;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
index 8c446e4..f6ac61a 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.query.optrule;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
index 3afd2ca..8f53084 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.rest.security;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/tomcat-ext/pom.xml
----------------------------------------------------------------------
diff --git a/tomcat-ext/pom.xml b/tomcat-ext/pom.xml
index 1a171ee..f6af642 100644
--- a/tomcat-ext/pom.xml
+++ b/tomcat-ext/pom.xml
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
[30/35] kylin git commit: KYLIN-1983 only keep apache-license in main
pom.xml
Posted by li...@apache.org.
KYLIN-1983 only keep apache-license in main pom.xml
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c59d63de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c59d63de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c59d63de
Branch: refs/heads/1.5.x-HBase1.x
Commit: c59d63de6fc404456bd59f4ca5d36e48fb251637
Parents: c5c8501
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 10:00:54 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
kylin-it/pom.xml | 63 ---------------------------------------
pom.xml | 11 +++++++
query/pom.xml | 68 ------------------------------------------
storage-hbase/pom.xml | 73 ----------------------------------------------
4 files changed, 11 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f2e03b6..e14fa7d 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -332,69 +332,6 @@
</plugins>
</build>
</profile>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- test data -->
- <exclude>src/test/**/*.dat</exclude>
- <exclude>src/test/**/*.expected</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d5925b9..38f9365 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1215,6 +1215,17 @@
<exclude>**/*.DEF</exclude>
<exclude>**/*.isl</exclude>
<exclude>**/*.isproj</exclude>
+
+ <!-- protobuf generated -->
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
+ </exclude>
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+ </exclude>
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+ </exclude>
</excludes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 0937b5a..8bb72ae 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -158,72 +158,4 @@
</exclusions>
</dependency>
</dependencies>
-
-
- <profiles>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- test data -->
- <exclude>src/test/**/*.dat</exclude>
- <exclude>src/test/**/*.expected</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index b21a931..b332354 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -150,77 +150,4 @@
</plugins>
</build>
- <profiles>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- protobuf generated -->
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
- </exclude>
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
- </exclude>
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
- </exclude>
- <exclude>**/src/test/resources/**</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
</project>
[16/35] kylin git commit: KYLIN-2005 Move all storage side behavior
hints to GTScanRequest
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index d9cef88..b0688b7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,115 +11,64 @@ public final class CubeVisitProtos {
public interface CubeVisitRequestOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // required string behavior = 1;
+ // required bytes gtScanRequest = 1;
/**
- * <code>required string behavior = 1;</code>
- */
- boolean hasBehavior();
- /**
- * <code>required string behavior = 1;</code>
- */
- java.lang.String getBehavior();
- /**
- * <code>required string behavior = 1;</code>
- */
- com.google.protobuf.ByteString
- getBehaviorBytes();
-
- // required bytes gtScanRequest = 2;
- /**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
boolean hasGtScanRequest();
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
com.google.protobuf.ByteString getGtScanRequest();
- // required bytes hbaseRawScan = 3;
+ // required bytes hbaseRawScan = 2;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
boolean hasHbaseRawScan();
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
com.google.protobuf.ByteString getHbaseRawScan();
- // required int32 rowkeyPreambleSize = 4;
+ // required int32 rowkeyPreambleSize = 3;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
boolean hasRowkeyPreambleSize();
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
int getRowkeyPreambleSize();
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>
getHbaseColumnsToGTList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index);
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
int getHbaseColumnsToGTCount();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index);
- // required int64 startTime = 6;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- boolean hasStartTime();
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- long getStartTime();
-
- // required int64 timeout = 7;
+ // required string kylinProperties = 5;
/**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- boolean hasTimeout();
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- long getTimeout();
-
- // required string kylinProperties = 8;
- /**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -127,7 +76,7 @@ public final class CubeVisitProtos {
*/
boolean hasKylinProperties();
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -135,7 +84,7 @@ public final class CubeVisitProtos {
*/
java.lang.String getKylinProperties();
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -197,44 +146,29 @@ public final class CubeVisitProtos {
}
case 10: {
bitField0_ |= 0x00000001;
- behavior_ = input.readBytes();
+ gtScanRequest_ = input.readBytes();
break;
}
case 18: {
bitField0_ |= 0x00000002;
- gtScanRequest_ = input.readBytes();
- break;
- }
- case 26: {
- bitField0_ |= 0x00000004;
hbaseRawScan_ = input.readBytes();
break;
}
- case 32: {
- bitField0_ |= 0x00000008;
+ case 24: {
+ bitField0_ |= 0x00000004;
rowkeyPreambleSize_ = input.readInt32();
break;
}
- case 42: {
- if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
+ case 34: {
+ if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>();
- mutable_bitField0_ |= 0x00000010;
+ mutable_bitField0_ |= 0x00000008;
}
hbaseColumnsToGT_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.PARSER, extensionRegistry));
break;
}
- case 48: {
- bitField0_ |= 0x00000010;
- startTime_ = input.readInt64();
- break;
- }
- case 56: {
- bitField0_ |= 0x00000020;
- timeout_ = input.readInt64();
- break;
- }
- case 66: {
- bitField0_ |= 0x00000040;
+ case 42: {
+ bitField0_ |= 0x00000008;
kylinProperties_ = input.readBytes();
break;
}
@@ -246,7 +180,7 @@ public final class CubeVisitProtos {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
- if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
+ if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
}
this.unknownFields = unknownFields.build();
@@ -785,196 +719,105 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required string behavior = 1;
- public static final int BEHAVIOR_FIELD_NUMBER = 1;
- private java.lang.Object behavior_;
- /**
- * <code>required string behavior = 1;</code>
- */
- public boolean hasBehavior() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public java.lang.String getBehavior() {
- java.lang.Object ref = behavior_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- behavior_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public com.google.protobuf.ByteString
- getBehaviorBytes() {
- java.lang.Object ref = behavior_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- behavior_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // required bytes gtScanRequest = 2;
- public static final int GTSCANREQUEST_FIELD_NUMBER = 2;
+ // required bytes gtScanRequest = 1;
+ public static final int GTSCANREQUEST_FIELD_NUMBER = 1;
private com.google.protobuf.ByteString gtScanRequest_;
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
- // required bytes hbaseRawScan = 3;
- public static final int HBASERAWSCAN_FIELD_NUMBER = 3;
+ // required bytes hbaseRawScan = 2;
+ public static final int HBASERAWSCAN_FIELD_NUMBER = 2;
private com.google.protobuf.ByteString hbaseRawScan_;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
- // required int32 rowkeyPreambleSize = 4;
- public static final int ROWKEYPREAMBLESIZE_FIELD_NUMBER = 4;
+ // required int32 rowkeyPreambleSize = 3;
+ public static final int ROWKEYPREAMBLESIZE_FIELD_NUMBER = 3;
private int rowkeyPreambleSize_;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public boolean hasRowkeyPreambleSize() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public int getRowkeyPreambleSize() {
return rowkeyPreambleSize_;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
- public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
+ public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
return hbaseColumnsToGT_.size();
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
return hbaseColumnsToGT_.get(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
return hbaseColumnsToGT_.get(index);
}
- // required int64 startTime = 6;
- public static final int STARTTIME_FIELD_NUMBER = 6;
- private long startTime_;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000010) == 0x00000010);
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public long getStartTime() {
- return startTime_;
- }
-
- // required int64 timeout = 7;
- public static final int TIMEOUT_FIELD_NUMBER = 7;
- private long timeout_;
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public boolean hasTimeout() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public long getTimeout() {
- return timeout_;
- }
-
- // required string kylinProperties = 8;
- public static final int KYLINPROPERTIES_FIELD_NUMBER = 8;
+ // required string kylinProperties = 5;
+ public static final int KYLINPROPERTIES_FIELD_NUMBER = 5;
private java.lang.Object kylinProperties_;
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public boolean hasKylinProperties() {
- return ((bitField0_ & 0x00000040) == 0x00000040);
+ return ((bitField0_ & 0x00000008) == 0x00000008);
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -995,7 +838,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -1016,13 +859,10 @@ public final class CubeVisitProtos {
}
private void initFields() {
- behavior_ = "";
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
rowkeyPreambleSize_ = 0;
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- startTime_ = 0L;
- timeout_ = 0L;
kylinProperties_ = "";
}
private byte memoizedIsInitialized = -1;
@@ -1030,10 +870,6 @@ public final class CubeVisitProtos {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
- if (!hasBehavior()) {
- memoizedIsInitialized = 0;
- return false;
- }
if (!hasGtScanRequest()) {
memoizedIsInitialized = 0;
return false;
@@ -1046,14 +882,6 @@ public final class CubeVisitProtos {
memoizedIsInitialized = 0;
return false;
}
- if (!hasStartTime()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasTimeout()) {
- memoizedIsInitialized = 0;
- return false;
- }
if (!hasKylinProperties()) {
memoizedIsInitialized = 0;
return false;
@@ -1066,28 +894,19 @@ public final class CubeVisitProtos {
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getBehaviorBytes());
+ output.writeBytes(1, gtScanRequest_);
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, gtScanRequest_);
+ output.writeBytes(2, hbaseRawScan_);
}
if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeBytes(3, hbaseRawScan_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- output.writeInt32(4, rowkeyPreambleSize_);
+ output.writeInt32(3, rowkeyPreambleSize_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
- output.writeMessage(5, hbaseColumnsToGT_.get(i));
- }
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- output.writeInt64(6, startTime_);
+ output.writeMessage(4, hbaseColumnsToGT_.get(i));
}
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- output.writeInt64(7, timeout_);
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
- output.writeBytes(8, getKylinPropertiesBytes());
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeBytes(5, getKylinPropertiesBytes());
}
getUnknownFields().writeTo(output);
}
@@ -1100,35 +919,23 @@ public final class CubeVisitProtos {
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getBehaviorBytes());
+ .computeBytesSize(1, gtScanRequest_);
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, gtScanRequest_);
+ .computeBytesSize(2, hbaseRawScan_);
}
if (((bitField0_ & 0x00000004) == 0x00000004)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(3, hbaseRawScan_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(4, rowkeyPreambleSize_);
+ .computeInt32Size(3, rowkeyPreambleSize_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(5, hbaseColumnsToGT_.get(i));
+ .computeMessageSize(4, hbaseColumnsToGT_.get(i));
}
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(6, startTime_);
- }
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(7, timeout_);
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(8, getKylinPropertiesBytes());
+ .computeBytesSize(5, getKylinPropertiesBytes());
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
@@ -1153,11 +960,6 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest) obj;
boolean result = true;
- result = result && (hasBehavior() == other.hasBehavior());
- if (hasBehavior()) {
- result = result && getBehavior()
- .equals(other.getBehavior());
- }
result = result && (hasGtScanRequest() == other.hasGtScanRequest());
if (hasGtScanRequest()) {
result = result && getGtScanRequest()
@@ -1175,16 +977,6 @@ public final class CubeVisitProtos {
}
result = result && getHbaseColumnsToGTList()
.equals(other.getHbaseColumnsToGTList());
- result = result && (hasStartTime() == other.hasStartTime());
- if (hasStartTime()) {
- result = result && (getStartTime()
- == other.getStartTime());
- }
- result = result && (hasTimeout() == other.hasTimeout());
- if (hasTimeout()) {
- result = result && (getTimeout()
- == other.getTimeout());
- }
result = result && (hasKylinProperties() == other.hasKylinProperties());
if (hasKylinProperties()) {
result = result && getKylinProperties()
@@ -1203,10 +995,6 @@ public final class CubeVisitProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasBehavior()) {
- hash = (37 * hash) + BEHAVIOR_FIELD_NUMBER;
- hash = (53 * hash) + getBehavior().hashCode();
- }
if (hasGtScanRequest()) {
hash = (37 * hash) + GTSCANREQUEST_FIELD_NUMBER;
hash = (53 * hash) + getGtScanRequest().hashCode();
@@ -1223,14 +1011,6 @@ public final class CubeVisitProtos {
hash = (37 * hash) + HBASECOLUMNSTOGT_FIELD_NUMBER;
hash = (53 * hash) + getHbaseColumnsToGTList().hashCode();
}
- if (hasStartTime()) {
- hash = (37 * hash) + STARTTIME_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getStartTime());
- }
- if (hasTimeout()) {
- hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getTimeout());
- }
if (hasKylinProperties()) {
hash = (37 * hash) + KYLINPROPERTIES_FIELD_NUMBER;
hash = (53 * hash) + getKylinProperties().hashCode();
@@ -1345,26 +1125,20 @@ public final class CubeVisitProtos {
public Builder clear() {
super.clear();
- behavior_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000002);
rowkeyPreambleSize_ = 0;
- bitField0_ = (bitField0_ & ~0x00000008);
+ bitField0_ = (bitField0_ & ~0x00000004);
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
hbaseColumnsToGTBuilder_.clear();
}
- startTime_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000020);
- timeout_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000040);
kylinProperties_ = "";
- bitField0_ = (bitField0_ & ~0x00000080);
+ bitField0_ = (bitField0_ & ~0x00000010);
return this;
}
@@ -1396,38 +1170,26 @@ public final class CubeVisitProtos {
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
to_bitField0_ |= 0x00000001;
}
- result.behavior_ = behavior_;
+ result.gtScanRequest_ = gtScanRequest_;
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000002;
}
- result.gtScanRequest_ = gtScanRequest_;
+ result.hbaseRawScan_ = hbaseRawScan_;
if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
to_bitField0_ |= 0x00000004;
}
- result.hbaseRawScan_ = hbaseRawScan_;
- if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
- to_bitField0_ |= 0x00000008;
- }
result.rowkeyPreambleSize_ = rowkeyPreambleSize_;
if (hbaseColumnsToGTBuilder_ == null) {
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
}
result.hbaseColumnsToGT_ = hbaseColumnsToGT_;
} else {
result.hbaseColumnsToGT_ = hbaseColumnsToGTBuilder_.build();
}
- if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
- to_bitField0_ |= 0x00000010;
- }
- result.startTime_ = startTime_;
- if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
- to_bitField0_ |= 0x00000020;
- }
- result.timeout_ = timeout_;
- if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
- to_bitField0_ |= 0x00000040;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000008;
}
result.kylinProperties_ = kylinProperties_;
result.bitField0_ = to_bitField0_;
@@ -1446,11 +1208,6 @@ public final class CubeVisitProtos {
public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other) {
if (other == org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.getDefaultInstance()) return this;
- if (other.hasBehavior()) {
- bitField0_ |= 0x00000001;
- behavior_ = other.behavior_;
- onChanged();
- }
if (other.hasGtScanRequest()) {
setGtScanRequest(other.getGtScanRequest());
}
@@ -1464,7 +1221,7 @@ public final class CubeVisitProtos {
if (!other.hbaseColumnsToGT_.isEmpty()) {
if (hbaseColumnsToGT_.isEmpty()) {
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
ensureHbaseColumnsToGTIsMutable();
hbaseColumnsToGT_.addAll(other.hbaseColumnsToGT_);
@@ -1477,7 +1234,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_.dispose();
hbaseColumnsToGTBuilder_ = null;
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
hbaseColumnsToGTBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getHbaseColumnsToGTFieldBuilder() : null;
@@ -1486,14 +1243,8 @@ public final class CubeVisitProtos {
}
}
}
- if (other.hasStartTime()) {
- setStartTime(other.getStartTime());
- }
- if (other.hasTimeout()) {
- setTimeout(other.getTimeout());
- }
if (other.hasKylinProperties()) {
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = other.kylinProperties_;
onChanged();
}
@@ -1502,10 +1253,6 @@ public final class CubeVisitProtos {
}
public final boolean isInitialized() {
- if (!hasBehavior()) {
-
- return false;
- }
if (!hasGtScanRequest()) {
return false;
@@ -1518,14 +1265,6 @@ public final class CubeVisitProtos {
return false;
}
- if (!hasStartTime()) {
-
- return false;
- }
- if (!hasTimeout()) {
-
- return false;
- }
if (!hasKylinProperties()) {
return false;
@@ -1552,192 +1291,118 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required string behavior = 1;
- private java.lang.Object behavior_ = "";
- /**
- * <code>required string behavior = 1;</code>
- */
- public boolean hasBehavior() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public java.lang.String getBehavior() {
- java.lang.Object ref = behavior_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- behavior_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public com.google.protobuf.ByteString
- getBehaviorBytes() {
- java.lang.Object ref = behavior_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- behavior_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder setBehavior(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- behavior_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder clearBehavior() {
- bitField0_ = (bitField0_ & ~0x00000001);
- behavior_ = getDefaultInstance().getBehavior();
- onChanged();
- return this;
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder setBehaviorBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- behavior_ = value;
- onChanged();
- return this;
- }
-
- // required bytes gtScanRequest = 2;
+ // required bytes gtScanRequest = 1;
private com.google.protobuf.ByteString gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public Builder setGtScanRequest(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000001;
gtScanRequest_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public Builder clearGtScanRequest() {
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
gtScanRequest_ = getDefaultInstance().getGtScanRequest();
onChanged();
return this;
}
- // required bytes hbaseRawScan = 3;
+ // required bytes hbaseRawScan = 2;
private com.google.protobuf.ByteString hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public Builder setHbaseRawScan(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000004;
+ bitField0_ |= 0x00000002;
hbaseRawScan_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public Builder clearHbaseRawScan() {
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000002);
hbaseRawScan_ = getDefaultInstance().getHbaseRawScan();
onChanged();
return this;
}
- // required int32 rowkeyPreambleSize = 4;
+ // required int32 rowkeyPreambleSize = 3;
private int rowkeyPreambleSize_ ;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public boolean hasRowkeyPreambleSize() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public int getRowkeyPreambleSize() {
return rowkeyPreambleSize_;
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public Builder setRowkeyPreambleSize(int value) {
- bitField0_ |= 0x00000008;
+ bitField0_ |= 0x00000004;
rowkeyPreambleSize_ = value;
onChanged();
return this;
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public Builder clearRowkeyPreambleSize() {
- bitField0_ = (bitField0_ & ~0x00000008);
+ bitField0_ = (bitField0_ & ~0x00000004);
rowkeyPreambleSize_ = 0;
onChanged();
return this;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_ =
java.util.Collections.emptyList();
private void ensureHbaseColumnsToGTIsMutable() {
- if (!((bitField0_ & 0x00000010) == 0x00000010)) {
+ if (!((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>(hbaseColumnsToGT_);
- bitField0_ |= 0x00000010;
+ bitField0_ |= 0x00000008;
}
}
@@ -1745,7 +1410,7 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder> hbaseColumnsToGTBuilder_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1755,7 +1420,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1765,7 +1430,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1775,7 +1440,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1792,7 +1457,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1806,7 +1471,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1822,7 +1487,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1839,7 +1504,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1853,7 +1518,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1867,7 +1532,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addAllHbaseColumnsToGT(
java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> values) {
@@ -1881,12 +1546,12 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder clearHbaseColumnsToGT() {
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
onChanged();
} else {
hbaseColumnsToGTBuilder_.clear();
@@ -1894,7 +1559,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder removeHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1907,14 +1572,14 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder getHbaseColumnsToGTBuilder(
int index) {
return getHbaseColumnsToGTFieldBuilder().getBuilder(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -1924,7 +1589,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
@@ -1935,14 +1600,14 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder() {
return getHbaseColumnsToGTFieldBuilder().addBuilder(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder(
int index) {
@@ -1950,7 +1615,7 @@ public final class CubeVisitProtos {
index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder>
getHbaseColumnsToGTBuilderList() {
@@ -1963,7 +1628,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>(
hbaseColumnsToGT_,
- ((bitField0_ & 0x00000010) == 0x00000010),
+ ((bitField0_ & 0x00000008) == 0x00000008),
getParentForChildren(),
isClean());
hbaseColumnsToGT_ = null;
@@ -1971,118 +1636,20 @@ public final class CubeVisitProtos {
return hbaseColumnsToGTBuilder_;
}
- // required int64 startTime = 6;
- private long startTime_ ;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public long getStartTime() {
- return startTime_;
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public Builder setStartTime(long value) {
- bitField0_ |= 0x00000020;
- startTime_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public Builder clearStartTime() {
- bitField0_ = (bitField0_ & ~0x00000020);
- startTime_ = 0L;
- onChanged();
- return this;
- }
-
- // required int64 timeout = 7;
- private long timeout_ ;
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public boolean hasTimeout() {
- return ((bitField0_ & 0x00000040) == 0x00000040);
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public long getTimeout() {
- return timeout_;
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public Builder setTimeout(long value) {
- bitField0_ |= 0x00000040;
- timeout_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public Builder clearTimeout() {
- bitField0_ = (bitField0_ & ~0x00000040);
- timeout_ = 0L;
- onChanged();
- return this;
- }
-
- // required string kylinProperties = 8;
+ // required string kylinProperties = 5;
private java.lang.Object kylinProperties_ = "";
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public boolean hasKylinProperties() {
- return ((bitField0_ & 0x00000080) == 0x00000080);
+ return ((bitField0_ & 0x00000010) == 0x00000010);
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2100,7 +1667,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2120,7 +1687,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2131,26 +1698,26 @@ public final class CubeVisitProtos {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = value;
onChanged();
return this;
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public Builder clearKylinProperties() {
- bitField0_ = (bitField0_ & ~0x00000080);
+ bitField0_ = (bitField0_ & ~0x00000010);
kylinProperties_ = getDefaultInstance().getKylinProperties();
onChanged();
return this;
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2161,7 +1728,7 @@ public final class CubeVisitProtos {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = value;
onChanged();
return this;
@@ -4521,26 +4088,25 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\370\001\n\020C" +
- "ubeVisitRequest\022\020\n\010behavior\030\001 \002(\t\022\025\n\rgtS" +
- "canRequest\030\002 \002(\014\022\024\n\014hbaseRawScan\030\003 \002(\014\022\032" +
- "\n\022rowkeyPreambleSize\030\004 \002(\005\0223\n\020hbaseColum" +
- "nsToGT\030\005 \003(\0132\031.CubeVisitRequest.IntList\022" +
- "\021\n\tstartTime\030\006 \002(\003\022\017\n\007timeout\030\007 \002(\003\022\027\n\017k" +
- "ylinProperties\030\010 \002(\t\032\027\n\007IntList\022\014\n\004ints\030" +
- "\001 \003(\005\"\321\002\n\021CubeVisitResponse\022\026\n\016compresse",
- "dRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitRe" +
- "sponse.Stats\032\372\001\n\005Stats\022\030\n\020serviceStartTi" +
- "me\030\001 \001(\003\022\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scan" +
- "nedRowCount\030\003 \001(\003\022\032\n\022aggregatedRowCount\030" +
- "\004 \001(\003\022\025\n\rsystemCpuLoad\030\005 \001(\001\022\036\n\026freePhys" +
- "icalMemorySize\030\006 \001(\001\022\031\n\021freeSwapSpaceSiz" +
- "e\030\007 \001(\001\022\020\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(" +
- "\t\022\026\n\016normalComplete\030\n \001(\0052F\n\020CubeVisitSe" +
- "rvice\0222\n\tvisitCube\022\021.CubeVisitRequest\032\022." +
- "CubeVisitResponseB`\nEorg.apache.kylin.st",
- "orage.hbase.cube.v2.coprocessor.endpoint" +
- ".generatedB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "/endpoint/protobuf/CubeVisit.proto\"\302\001\n\020C" +
+ "ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
+ "\n\014hbaseRawScan\030\002 \002(\014\022\032\n\022rowkeyPreambleSi" +
+ "ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
+ "eVisitRequest.IntList\022\027\n\017kylinProperties" +
+ "\030\005 \002(\t\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\321\002\n\021Cube" +
+ "VisitResponse\022\026\n\016compressedRows\030\001 \002(\014\022\'\n" +
+ "\005stats\030\002 \002(\0132\030.CubeVisitResponse.Stats\032\372",
+ "\001\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026\n\016se" +
+ "rviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 " +
+ "\001(\003\022\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\rsyste" +
+ "mCpuLoad\030\005 \001(\001\022\036\n\026freePhysicalMemorySize" +
+ "\030\006 \001(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020\n\010hos" +
+ "tname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016normalCom" +
+ "plete\030\n \001(\0052F\n\020CubeVisitService\0222\n\tvisit" +
+ "Cube\022\021.CubeVisitRequest\032\022.CubeVisitRespo" +
+ "nseB`\nEorg.apache.kylin.storage.hbase.cu" +
+ "be.v2.coprocessor.endpoint.generatedB\017Cu",
+ "beVisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4552,7 +4118,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "Behavior", "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "StartTime", "Timeout", "KylinProperties", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index e1de070..c84f4f3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -30,14 +30,11 @@ option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
message CubeVisitRequest {
- required string behavior = 1;
- required bytes gtScanRequest = 2;
- required bytes hbaseRawScan = 3;
- required int32 rowkeyPreambleSize = 4;
- repeated IntList hbaseColumnsToGT = 5;
- required int64 startTime = 6;//when client start the request
- required int64 timeout = 7;//how long client will wait
- required string kylinProperties = 8; // kylin properties
+ required bytes gtScanRequest = 1;
+ required bytes hbaseRawScan = 2;
+ required int32 rowkeyPreambleSize = 3;
+ repeated IntList hbaseColumnsToGT = 4;
+ required string kylinProperties = 5; // kylin properties
message IntList {
repeated int32 ints = 1;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index f8e2644..390930a 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -40,7 +40,7 @@ import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -121,7 +121,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -170,7 +170,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[21/35] kylin git commit: remove unnecessary raw measure cond
Posted by li...@apache.org.
remove unnecessary raw measure cond
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a05f1114
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a05f1114
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a05f1114
Branch: refs/heads/1.5.x-HBase1.x
Commit: a05f11146c9edd4ba18618f38fd13c9b7e4806ce
Parents: 942406b
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 23:45:17 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 23:45:17 2016 +0800
----------------------------------------------------------------------
.../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 3 ---
.../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 3 ---
2 files changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 7de2ae2..d185175 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -234,9 +234,6 @@
"item_count_sum",
"SITE_EXTENDED_1",
"SITE_EXTENDED_2",
- "CAL_DT_RAW",
- "LSTG_FORMAT_NAME_RAW",
- "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 4270aab..2aea1a8 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -266,9 +266,6 @@
"gmv_max",
"trans_cnt",
"item_count_sum",
- "CAL_DT_RAW",
- "LSTG_FORMAT_NAME_RAW",
- "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
[07/35] kylin git commit: more test queries and minor refactor
Posted by li...@apache.org.
more test queries and minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ddeb7452
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ddeb7452
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ddeb7452
Branch: refs/heads/1.5.x-HBase1.x
Commit: ddeb7452a63279311b74fcfb5f7ed0252e3d54fd
Parents: fc7281b
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Sep 6 16:09:59 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 8 10:15:08 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 63 +-------
.../filter/BuiltInFunctionTupleFilter.java | 12 +-
.../filter/EvaluatableFunctionTupleFilter.java | 151 +++++++++++++++++++
.../filter/EvaluatableLikeFunction.java | 114 --------------
.../kylin/metadata/filter/TupleFilter.java | 2 +-
.../metadata/filter/TupleFilterSerializer.java | 4 +-
.../metadata/filter/function/BuiltInMethod.java | 9 ++
.../test_kylin_inner_join_model_desc.json | 103 ++++++++-----
.../test_kylin_inner_join_view_model_desc.json | 24 ++-
.../test_kylin_left_join_model_desc.json | 31 ++--
.../test_kylin_left_join_view_model_desc.json | 24 ++-
.../apache/kylin/query/ITKylinQueryTest.java | 25 +--
.../src/test/resources/query/sql/query45.sql | 24 ---
.../src/test/resources/query/sql/query46.sql | 19 ---
.../src/test/resources/query/sql/query47.sql | 19 ---
.../src/test/resources/query/sql/query48.sql | 19 ---
.../src/test/resources/query/sql/query55.sql | 19 ---
.../src/test/resources/query/sql/query83.sql | 33 ++++
.../src/test/resources/query/sql/query84.sql | 33 ++++
.../src/test/resources/query/sql/query92.sql | 6 +-
.../src/test/resources/query/sql/query93.sql | 6 +-
.../src/test/resources/query/sql/query94.sql | 6 +-
.../src/test/resources/query/sql/query95.sql | 6 +-
.../test/resources/query/sql_like/query04.sql | 22 +++
.../test/resources/query/sql_like/query10.sql | 13 ++
.../test/resources/query/sql_like/query15.sql | 13 ++
.../test/resources/query/sql_like/query16.sql | 13 ++
.../test/resources/query/sql_like/query17.sql | 13 ++
.../test/resources/query/sql_lookup/query45.sql | 24 +++
.../test/resources/query/sql_lookup/query46.sql | 19 +++
.../test/resources/query/sql_lookup/query47.sql | 19 +++
.../test/resources/query/sql_lookup/query48.sql | 19 +++
.../test/resources/query/sql_lookup/query55.sql | 19 +++
.../test/resources/query/sql_raw/query21.sql | 24 +++
.../test/resources/query/sql_raw/query22.sql | 24 +++
.../test/resources/query/sql_raw/query23.sql | 24 +++
.../test/resources/query/sql_raw/query24.sql | 24 +++
.../test/resources/query/sql_raw/query25.sql | 24 +++
.../query/sql_raw/query26.sql.disabled | 24 +++
.../kylin/query/relnode/OLAPAggregateRel.java | 15 +-
.../apache/kylin/query/relnode/OLAPContext.java | 7 +
.../apache/kylin/query/relnode/OLAPJoinRel.java | 40 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 6 +-
.../apache/kylin/query/routing/Candidate.java | 5 +-
44 files changed, 753 insertions(+), 390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 7406e81..8d310a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -18,7 +18,6 @@
package org.apache.kylin.gridtable;
-import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
@@ -26,7 +25,6 @@ import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -108,11 +106,11 @@ public class GTUtil {
};
}
- private static class GTConvertDecorator implements TupleFilterSerializer.Decorator {
- private final Set<TblColRef> unevaluatableColumnCollector;
- private final List<TblColRef> colMapping;
- private final GTInfo info;
- private final boolean encodeConstants;
+ protected static class GTConvertDecorator implements TupleFilterSerializer.Decorator {
+ protected final Set<TblColRef> unevaluatableColumnCollector;
+ protected final List<TblColRef> colMapping;
+ protected final GTInfo info;
+ protected final boolean encodeConstants;
public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, List<TblColRef> colMapping, GTInfo info, boolean encodeConstants) {
this.unevaluatableColumnCollector = unevaluatableColumnCollector;
@@ -152,19 +150,12 @@ public class GTUtil {
if (encodeConstants && filter instanceof CompareTupleFilter) {
return encodeConstants((CompareTupleFilter) filter);
}
- if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
- if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
- return encodeConstants((BuiltInFunctionTupleFilter) filter);
- } else {
- throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
- }
- }
return filter;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
+ protected TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
// extract ColumnFilter & ConstantFilter
TblColRef externalCol = oldCompareFilter.getColumn();
@@ -258,49 +249,9 @@ public class GTUtil {
return result;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
- // extract ColumnFilter & ConstantFilter
- TblColRef externalCol = funcFilter.getColumn();
-
- if (externalCol == null) {
- return funcFilter;
- }
-
- Collection constValues = funcFilter.getConstantTupleFilter().getValues();
- if (constValues == null || constValues.isEmpty()) {
- return funcFilter;
- }
-
- BuiltInFunctionTupleFilter newFuncFilter;
- try {
- newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
- } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
- throw new RuntimeException(e);
- }
- newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
-
- int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
-
- ByteArray code;
-
- // translate constant into code
- Set newValues = Sets.newHashSet();
- for (Object value : constValues) {
- code = translate(col, value, 0);
- if (code == null) {
- throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
- }
- newValues.add(code);
- }
- newFuncFilter.addChild(new ConstantTupleFilter(newValues));
-
- return newFuncFilter;
- }
-
transient ByteBuffer buf;
- private ByteArray translate(int col, Object value, int roundingFlag) {
+ protected ByteArray translate(int col, Object value, int roundingFlag) {
try {
buf.clear();
info.codeSystem.encodeColumnValue(col, value, roundingFlag, buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 40afb18..734b374 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -43,6 +43,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
protected ConstantTupleFilter constantTupleFilter;
protected int colPosition;
+ protected int constantPosition;
protected Method method;
protected List<Serializable> methodParams;
protected boolean isValidFunc = false;
@@ -69,6 +70,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
return constantTupleFilter;
}
+ public TupleFilter getColumnContainerFilter() {
+ return columnContainerFilter;
+ }
+
public TblColRef getColumn() {
if (columnContainerFilter == null)
return null;
@@ -81,10 +86,6 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
}
- public boolean hasNested() {
- return (columnContainerFilter != null && columnContainerFilter instanceof BuiltInFunctionTupleFilter);
- }
-
public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
if (columnContainerFilter instanceof ColumnTupleFilter)
methodParams.set(colPosition, (Serializable) input);
@@ -107,6 +108,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
this.constantTupleFilter = (ConstantTupleFilter) child;
Serializable constVal = (Serializable) child.getValues().iterator().next();
try {
+ constantPosition = methodParams.size();
Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
if (!Primitives.isWrapperType(clazz))
methodParams.add(constVal);
@@ -131,7 +133,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
new file mode 100644
index 0000000..ff24172
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.StringSerializer;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
+import com.google.common.collect.Lists;
+
+public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter {
+
+ private boolean constantsInitted = false;
+
+ //about non-like
+ private List<Object> values;
+ private Object tupleValue;
+
+ public EvaluatableFunctionTupleFilter(String name) {
+ super(name, FilterOperatorEnum.EVAL_FUNC);
+ values = Lists.newArrayListWithCapacity(1);
+ values.add(null);
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
+
+ // extract tuple value
+ Object tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ if (!isConstant(filter)) {
+ filter.evaluate(tuple, cs);
+ tupleValue = filter.getValues().iterator().next();
+ }
+ }
+
+ TblColRef tblColRef = this.getColumn();
+ DataType strDataType = DataType.getType("string");
+ if (tblColRef.getType() != strDataType) {
+ throw new IllegalStateException("Only String type is allow in BuiltInFunction");
+ }
+ ByteArray valueByteArray = (ByteArray) tupleValue;
+ StringSerializer serializer = new StringSerializer(strDataType);
+ String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
+
+ try {
+ if (isLikeFunction()) {
+ return (Boolean) invokeFunction(value);
+ } else {
+ this.tupleValue = invokeFunction(value);
+ //convert back to ByteArray format because the outer EvaluatableFunctionTupleFilter assumes input as ByteArray
+ ByteBuffer buffer = ByteBuffer.allocate(valueByteArray.length() * 2);
+ serializer.serialize((String) this.tupleValue, buffer);
+ this.tupleValue = new ByteArray(buffer.array(), 0, buffer.position());
+
+ return true;
+ }
+ } catch (InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Collection<?> getValues() {
+ this.values.set(0, tupleValue);
+ return values;
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ if (!isValid()) {
+ throw new IllegalStateException("must be valid");
+ }
+ BytesUtil.writeUTFString(name, buffer);
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ private boolean isConstant(TupleFilter filter) {
+ return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+ }
+
+ @Override
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (isLikeFunction())
+ initConstants();
+ return super.invokeFunction(input);
+ }
+
+ private void initConstants() {
+ if (constantsInitted) {
+ return;
+ }
+ //will replace the ByteArray pattern to String type
+ ByteArray byteArray = (ByteArray) methodParams.get(constantPosition);
+ StringSerializer s = new StringSerializer(DataType.getType("string"));
+ String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
+ //TODO
+ //pattern = pattern.toLowerCase();//to remove upper case
+ methodParams.set(constantPosition, pattern);
+ constantsInitted = true;
+ }
+
+ //even for "tolower(s)/toupper(s)/substring(like) like pattern", the like pattern can be used for index searching
+ public String getLikePattern() {
+ if (!isLikeFunction()) {
+ return null;
+ }
+
+ initConstants();
+ return (String) methodParams.get(1);
+ }
+
+ public boolean isLikeFunction() {
+ return "like".equalsIgnoreCase(this.getName());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
deleted file mode 100644
index 28e544b..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-/**
- * typically like will be handled by BuiltInFunctionTupleFilter rather than this
- */
-public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
-
- private boolean patternExtracted;
-
- public EvaluatableLikeFunction(String name) {
- super(name, FilterOperatorEnum.LIKE);
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
- // extract tuple value
- Object tupleValue = null;
- for (TupleFilter filter : this.children) {
- if (!isConstant(filter)) {
- filter.evaluate(tuple, cs);
- tupleValue = filter.getValues().iterator().next();
- }
- }
-
- // consider null case
- if (cs.isNull(tupleValue)) {
- return false;
- }
-
- ByteArray valueByteArray = (ByteArray) tupleValue;
- StringSerializer serializer = new StringSerializer(DataType.getType("string"));
- String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
- try {
- return (Boolean) invokeFunction(value);
- } catch (InvocationTargetException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- if (!isValid()) {
- throw new IllegalStateException("must be valid");
- }
- if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1) == null) {
- throw new IllegalArgumentException("bad methodParams: " + methodParams);
- }
- BytesUtil.writeUTFString(name, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- private boolean isConstant(TupleFilter filter) {
- return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
- }
-
- @Override
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (!patternExtracted) {
- this.getLikePattern();
- }
- return super.invokeFunction(input);
- }
-
- //will replace the ByteArray pattern to String type
- public String getLikePattern() {
- if (!patternExtracted) {
- ByteArray byteArray = (ByteArray) methodParams.get(1);
- StringSerializer s = new StringSerializer(DataType.getType("string"));
- String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
- methodParams.set(1, pattern);
- patternExtracted = true;
- }
- return (String) methodParams.get(1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 2fb4e1f..900cc35 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -38,7 +38,7 @@ import com.google.common.collect.Maps;
public abstract class TupleFilter {
public enum FilterOperatorEnum {
- EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), LIKE(37);
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37);
private final int value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 0aca9a1..04984f2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -188,8 +188,8 @@ public class TupleFilterSerializer {
case FUNCTION:
filter = new BuiltInFunctionTupleFilter(null);
break;
- case LIKE:
- filter = new EvaluatableLikeFunction(null);
+ case EVAL_FUNC:
+ filter = new EvaluatableFunctionTupleFilter(null);
break;
case MASSIN:
filter = new MassInTupleFilter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index 7b241cc..2f28fae 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -47,6 +47,9 @@ public enum BuiltInMethod {
/** SQL {@code LIKE} function. */
public static boolean like(String s, String pattern) {
+ if (s == null)
+ return false;
+
final String regex = Like.sqlToRegexLike(pattern, null);
return Pattern.matches(regex, s);
}
@@ -95,16 +98,22 @@ public enum BuiltInMethod {
/** SQL SUBSTRING(string FROM ... FOR ...) function. */
public static String substring(String s, int from, int for_) {
+ if (s == null)
+ return null;
return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
}
/** SQL UPPER(string) function. */
public static String upper(String s) {
+ if (s == null)
+ return null;
return s.toUpperCase();
}
/** SQL LOWER(string) function. */
public static String lower(String s) {
+ if (s == null)
+ return null;
return s.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index cb43d6f..6e444e5 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -1,35 +1,58 @@
{
- "uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
- "name" : "test_kylin_inner_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
+ "uuid": "ff527b94-f860-44c3-8452-93b17774c647",
+ "name": "test_kylin_inner_join_model_desc",
+ "lookups": [
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "LEAF_CATEG_ID",
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LEAF_CATEG_ID",
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
+ }
}
- } ],
+ ],
"dimensions": [
{
"table": "default.test_kylin_fact",
@@ -81,16 +104,16 @@
}
],
"metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID"
+ "PRICE",
+ "ITEM_COUNT",
+ "SELLER_ID"
],
- "last_modified" : 1422435345352,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
+ "last_modified": 1422435345352,
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
index acdcab8..621eccc 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "inner",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "inner",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "inner",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index 735ca05..dab99f9 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "left",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "left",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
@@ -45,11 +57,12 @@
{
"table": "default.test_kylin_fact",
"columns": [
- "TRANS_ID",
- "CAL_DT",
"lstg_format_name",
"LSTG_SITE_ID",
"SLR_SEGMENT_CD",
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
"SELLER_ID"
]
},
@@ -83,7 +96,7 @@
]
},
{
- "table": "EDW.TEST_CAL_DT",
+ "table": "edw.test_cal_dt",
"columns": [
"cal_dt",
"week_beg_dt"
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
index a4af260..819b8b0 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "left",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "left",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4657a5c..375b198 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -55,12 +55,13 @@ public class ITKylinQueryTest extends KylinTestBase {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
Candidate.setPriorities(priorities);
- joinType = "inner";
+ joinType = "left";
setupAll();
-
+
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
@@ -192,18 +193,25 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
- public void testDimDistinctCountQuery() throws Exception {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
+ public void testTopNQuery() throws Exception {
+ if ("left".equalsIgnoreCase(joinType)) {
+ this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true);
+ }
}
@Test
public void testPreciselyDistinctCountQuery() throws Exception {
if ("left".equalsIgnoreCase(joinType)) {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/temp", null, true);
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_precisely", null, true);
}
}
@Test
+ public void testDimDistinctCountQuery() throws Exception {
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
+ }
+
+ @Test
public void testStreamingTableQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_streaming", null, true);
}
@@ -276,13 +284,6 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
- public void testTopNQuery() throws Exception {
- if ("left".equalsIgnoreCase(joinType)) {
- this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true);
- }
- }
-
- @Test
public void testRawQuery() throws Exception {
this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_raw", null, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query45.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query45.sql b/kylin-it/src/test/resources/query/sql/query45.sql
deleted file mode 100644
index ea964ae..0000000
--- a/kylin-it/src/test/resources/query/sql/query45.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_cal_dt
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query46.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query46.sql b/kylin-it/src/test/resources/query/sql/query46.sql
deleted file mode 100644
index 3bfe9d9..0000000
--- a/kylin-it/src/test/resources/query/sql/query46.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from test_category_groupings
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query47.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query47.sql b/kylin-it/src/test/resources/query/sql/query47.sql
deleted file mode 100644
index cbd2c6d..0000000
--- a/kylin-it/src/test/resources/query/sql/query47.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_seller_type_dim
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query48.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query48.sql b/kylin-it/src/test/resources/query/sql/query48.sql
deleted file mode 100644
index 54ddb31..0000000
--- a/kylin-it/src/test/resources/query/sql/query48.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_sites
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query55.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query55.sql b/kylin-it/src/test/resources/query/sql/query55.sql
deleted file mode 100644
index 346a7d7..0000000
--- a/kylin-it/src/test/resources/query/sql/query55.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as c from edw.test_cal_dt as test_cal_dt where extract(YEAR from test_cal_dt.cal_dt) = 2012
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query83.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query83.sql b/kylin-it/src/test/resources/query/sql/query83.sql
new file mode 100644
index 0000000..1fbbd33
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query83.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(1) as TRANS_CNT
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and
+ test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ on test_kylin_fact.lstg_site_id = test_sites.site_id
+ left JOIN edw.test_seller_type_dim as test_seller_type_dim
+ on test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd
+ where test_kylin_fact.lstg_format_name='FP-GTC'
+ and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ and test_kylin_fact.cal_dt between DATE '2013-06-01' and DATE '2013-09-01'
+ group by test_cal_dt.week_beg_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query84.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query84.sql b/kylin-it/src/test/resources/query/sql/query84.sql
new file mode 100644
index 0000000..2560465
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query84.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(1) as TRANS_CNT
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and
+ test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ on test_kylin_fact.lstg_site_id = test_sites.site_id
+ left JOIN edw.test_seller_type_dim as test_seller_type_dim
+ on test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd
+ where test_kylin_fact.lstg_format_name='FP-GTC'
+ and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ and test_cal_dt.cal_dt between DATE '2013-06-01' and DATE '2013-09-01'
+ group by test_cal_dt.week_beg_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query92.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query92.sql b/kylin-it/src/test/resources/query/sql/query92.sql
index e551a45..ebc07fe 100644
--- a/kylin-it/src/test/resources/query/sql/query92.sql
+++ b/kylin-it/src/test/resources/query/sql/query92.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('', 'a')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query93.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query93.sql b/kylin-it/src/test/resources/query/sql/query93.sql
index cc6dca5..085e0e6 100644
--- a/kylin-it/src/test/resources/query/sql/query93.sql
+++ b/kylin-it/src/test/resources/query/sql/query93.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name is not null
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query94.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query94.sql b/kylin-it/src/test/resources/query/sql/query94.sql
index c7899fd..b39d163 100644
--- a/kylin-it/src/test/resources/query/sql/query94.sql
+++ b/kylin-it/src/test/resources/query/sql/query94.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('Unknown')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query95.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query95.sql b/kylin-it/src/test/resources/query/sql/query95.sql
index 578b93f..fd94dcc 100644
--- a/kylin-it/src/test/resources/query/sql/query95.sql
+++ b/kylin-it/src/test/resources/query/sql/query95.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query04.sql b/kylin-it/src/test/resources/query/sql_like/query04.sql
new file mode 100644
index 0000000..faf5ca3
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query04.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='auction' and substring(lstg_format_name,1,3) in ('Auc') and upper(lstg_format_name) > 'AAAA' and
+upper(lstg_format_name) like '%UC%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='Auctiona'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query10.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query10.sql b/kylin-it/src/test/resources/query/sql_like/query10.sql
new file mode 100644
index 0000000..21632f4
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query10.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where USER_DEFINED_FIELD3 like '%Video Game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query15.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query15.sql b/kylin-it/src/test/resources/query/sql_like/query15.sql
new file mode 100644
index 0000000..85418c7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query15.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lower(USER_DEFINED_FIELD3) like '%Video Game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query16.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query16.sql b/kylin-it/src/test/resources/query/sql_like/query16.sql
new file mode 100644
index 0000000..2a484ef
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query16.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lower(USER_DEFINED_FIELD3) like '%video game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query17.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query17.sql b/kylin-it/src/test/resources/query/sql_like/query17.sql
new file mode 100644
index 0000000..c6dd1ea
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query17.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where upper(USER_DEFINED_FIELD3) like '%VIDEO GAME%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query45.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query45.sql b/kylin-it/src/test/resources/query/sql_lookup/query45.sql
new file mode 100644
index 0000000..ea964ae
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query45.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_cal_dt
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query46.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query46.sql b/kylin-it/src/test/resources/query/sql_lookup/query46.sql
new file mode 100644
index 0000000..3bfe9d9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query46.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from test_category_groupings
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query47.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query47.sql b/kylin-it/src/test/resources/query/sql_lookup/query47.sql
new file mode 100644
index 0000000..cbd2c6d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query47.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_seller_type_dim
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query48.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query48.sql b/kylin-it/src/test/resources/query/sql_lookup/query48.sql
new file mode 100644
index 0000000..54ddb31
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query48.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_sites
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query55.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query55.sql b/kylin-it/src/test/resources/query/sql_lookup/query55.sql
new file mode 100644
index 0000000..346a7d7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query55.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as c from edw.test_cal_dt as test_cal_dt where extract(YEAR from test_cal_dt.cal_dt) = 2012
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query21.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query21.sql b/kylin-it/src/test/resources/query/sql_raw/query21.sql
new file mode 100644
index 0000000..4905e0f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query21.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query22.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query22.sql b/kylin-it/src/test/resources/query/sql_raw/query22.sql
new file mode 100644
index 0000000..d603a25
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query22.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query23.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query23.sql b/kylin-it/src/test/resources/query/sql_raw/query23.sql
new file mode 100644
index 0000000..89d61db
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query23.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query24.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query24.sql b/kylin-it/src/test/resources/query/sql_raw/query24.sql
new file mode 100644
index 0000000..46c7329
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query24.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query25.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query25.sql b/kylin-it/src/test/resources/query/sql_raw/query25.sql
new file mode 100644
index 0000000..175124f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query25.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled b/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
new file mode 100644
index 0000000..cab92d0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,test_kylin_fact.CAL_DT,test_category_groupings.LEAF_CATEG_ID,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index ba74c74..f55c86f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.Sets;
import org.apache.calcite.adapter.enumerable.EnumerableAggregate;
import org.apache.calcite.adapter.enumerable.EnumerableConvention;
import org.apache.calcite.adapter.enumerable.EnumerableRel;
@@ -67,6 +66,7 @@ import org.apache.kylin.query.schema.OLAPTable;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
*/
@@ -133,8 +133,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
if (getGroupType() == Group.SIMPLE) {
cost = super.computeSelfCost(planner, mq).multiplyBy(.05);
} else {
- cost = super.computeSelfCost(planner, mq).multiplyBy(.05).plus(planner.getCost(getInput(), mq))
- .multiplyBy(groupSets.size() * 1.5);
+ cost = super.computeSelfCost(planner, mq).multiplyBy(.05).plus(planner.getCost(getInput(), mq)).multiplyBy(groupSets.size() * 1.5);
}
return cost;
}
@@ -174,7 +173,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// Add group column indicators
if (indicator) {
final Set<String> containedNames = Sets.newHashSet();
- for (TblColRef groupCol: groups) {
+ for (TblColRef groupCol : groups) {
String base = "i$" + groupCol.getName();
String name = base;
int i = 0;
@@ -356,6 +355,14 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
private AggregateCall rewriteAggregateCall(AggregateCall aggCall, FunctionDesc func) {
+
+ //if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
+ // which do not exist at all
+ if (!this.context.hasPrecalculatedFields() && func.needRewriteField()) {
+ logger.info(func + "skip rewriteAggregateCall because no pre-aggregated field available");
+ return aggCall;
+ }
+
// rebuild parameters
List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
if (func.needRewriteField()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index c2e1b88..41a3b4d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
@@ -38,6 +39,7 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import com.google.common.collect.Lists;
@@ -146,6 +148,10 @@ public class OLAPContext {
return sqlDigest;
}
+ public boolean hasPrecalculatedFields() {
+ return realization instanceof CubeInstance || realization instanceof HybridInstance;
+ }
+
public void resetSQLDigest() {
this.sqlDigest = null;
}
@@ -167,6 +173,7 @@ public class OLAPContext {
sortOrders.add(order);
}
}
+
public interface IAccessController {
/*
* @return {TupleFilter} if the filter condition exists
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6dbb81a..2a143fb 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -48,9 +48,9 @@ import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
@@ -166,6 +166,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) {
TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue());
this.context.groupByColumns.add(fromCol);
+ this.context.allColumns.add(fromCol);
}
joinCol.clear();
}
@@ -295,26 +296,29 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
this.rowType = this.deriveRowType();
if (this.isTopJoin && RewriteImplementor.needRewrite(this.context)) {
- // find missed rewrite fields
- int paramIndex = this.rowType.getFieldList().size();
- List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
- for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
- String fieldName = rewriteField.getKey();
- if (this.rowType.getField(fieldName, true, false) == null) {
- RelDataType fieldType = rewriteField.getValue();
- RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, paramIndex++, fieldType);
- newFieldList.add(newField);
+ if (this.context.hasPrecalculatedFields()) {
+
+ // find missed rewrite fields
+ int paramIndex = this.rowType.getFieldList().size();
+ List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
+ for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
+ String fieldName = rewriteField.getKey();
+ if (this.rowType.getField(fieldName, true, false) == null) {
+ RelDataType fieldType = rewriteField.getValue();
+ RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, paramIndex++, fieldType);
+ newFieldList.add(newField);
+ }
}
- }
- // rebuild row type
- FieldInfoBuilder fieldInfo = getCluster().getTypeFactory().builder();
- fieldInfo.addAll(this.rowType.getFieldList());
- fieldInfo.addAll(newFieldList);
- this.rowType = getCluster().getTypeFactory().createStructType(fieldInfo);
+ // rebuild row type
+ FieldInfoBuilder fieldInfo = getCluster().getTypeFactory().builder();
+ fieldInfo.addAll(this.rowType.getFieldList());
+ fieldInfo.addAll(newFieldList);
+ this.rowType = getCluster().getTypeFactory().createStructType(fieldInfo);
- // rebuild columns
- this.columnRowType = this.buildColumnRowType();
+ // rebuild columns
+ this.columnRowType = this.buildColumnRowType();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 0a8a15f..db6ec2d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -37,9 +37,9 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -97,7 +97,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
boolean hasRexOver = RexOver.containsOver(getProjects(), null);
- return super.computeSelfCost(planner, mq).multiplyBy(.05).multiplyBy(getProjects().size() * (hasRexOver ? 50 : 1));
+ return super.computeSelfCost(planner, mq).multiplyBy(.05).multiplyBy(getProjects().size() * (hasRexOver ? 50 : 1));
}
@Override
@@ -244,7 +244,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
this.rewriting = true;
// project before join or is just after OLAPToEnumerableConverter
- if (!RewriteImplementor.needRewrite(this.context) || (this.hasJoin && !this.afterJoin) || this.afterAggregate) {
+ if (!RewriteImplementor.needRewrite(this.context) || (this.hasJoin && !this.afterJoin) || this.afterAggregate || !(this.context.hasPrecalculatedFields())) {
this.columnRowType = this.buildColumnRowType();
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index 9ea8961..bc17721 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.routing;
+import java.util.Collections;
import java.util.Map;
import org.apache.kylin.metadata.realization.CapabilityResult;
@@ -40,12 +41,12 @@ public class Candidate implements Comparable<Candidate> {
/** for test only */
public static void setPriorities(Map<RealizationType, Integer> priorities) {
- PRIORITIES = priorities;
+ PRIORITIES = Collections.unmodifiableMap(priorities);
}
/** for test only */
public static void restorePriorities() {
- PRIORITIES = DEFAULT_PRIORITIES;
+ PRIORITIES = Collections.unmodifiableMap(DEFAULT_PRIORITIES);
}
// ============================================================================
[06/35] kylin git commit: KYLIN-1789 fix wrong test cube metadata
Posted by li...@apache.org.
KYLIN-1789 fix wrong test cube metadata
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc7281b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc7281b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc7281b2
Branch: refs/heads/1.5.x-HBase1.x
Commit: fc7281b21df6bf48b10999bb10e270207540bb18
Parents: 9777878
Author: shaofengshi <sh...@apache.org>
Authored: Thu Sep 8 07:58:56 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Sep 8 07:58:56 2016 +0800
----------------------------------------------------------------------
.../model_desc/test_kylin_inner_join_model_desc.json | 1 -
.../model_desc/test_kylin_inner_join_view_model_desc.json | 9 ---------
.../model_desc/test_kylin_left_join_model_desc.json | 8 +++++++-
.../model_desc/test_kylin_left_join_view_model_desc.json | 9 ---------
4 files changed, 7 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index f5333b0..cb43d6f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
"name" : "test_kylin_inner_join_model_desc",
"lookups" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
index 54e9ffb..acdcab8 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
@@ -1,17 +1,8 @@
{
-
"uuid": "8b184ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_inner_join_view_model_desc",
"lookups": [
{
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "inner",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
- }
- },
- {
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "inner",
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index bedb167..735ca05 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid": "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_left_join_model_desc",
"lookups": [
@@ -82,6 +81,13 @@
"seller_type_cd",
"seller_type_desc"
]
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
}
],
"metrics": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
index 7c009a1..a4af260 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
@@ -1,17 +1,8 @@
{
-
"uuid": "9c184ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_left_join_view_model_desc",
"lookups": [
{
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
- }
- },
- {
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "left",
[22/35] kylin git commit: Revert "remove unnecessary raw measure"
Posted by li...@apache.org.
Revert "remove unnecessary raw measure"
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7a3fdf5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7a3fdf5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7a3fdf5
Branch: refs/heads/1.5.x-HBase1.x
Commit: d7a3fdf57acb6fbabf94669aaca869e47d89aa13
Parents: a05f111
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Sep 10 14:40:25 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Sep 10 14:42:46 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/topn/TopNMeasureType.java | 2 +
.../test_case_data/localmeta/cube_desc/ssb.json | 409 ++++++-------
.../test_kylin_cube_with_slr_desc.json | 389 +++++-------
...st_kylin_cube_with_view_inner_join_desc.json | 388 +++++-------
...est_kylin_cube_with_view_left_join_desc.json | 388 +++++-------
.../test_kylin_cube_without_slr_desc.json | 61 +-
...t_kylin_cube_without_slr_left_join_desc.json | 584 +++++++++----------
.../test_streaming_table_cube_desc.json | 245 ++++----
8 files changed, 1048 insertions(+), 1418 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 0756056..01eb90c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -274,9 +274,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (sum.isSum() == false)
return false;
+
if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0)
return false;
+
TblColRef sumCol = sum.getParameter().getColRefs().get(0);
return sumCol.equals(topnNumCol);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index 4903979..d3ea10b 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -1,256 +1,179 @@
{
- "uuid": "5c44df30-daec-486e-af90-927bf7851057",
- "name": "ssb",
- "description": "",
- "dimensions": [
- {
- "name": "SSB.PART_DERIVED",
- "table": "SSB.PART",
- "column": null,
- "derived": [
- "P_MFGR",
- "P_CATEGORY",
- "P_BRAND"
- ]
- },
- {
- "name": "C_CITY",
- "table": "SSB.CUSTOMER",
- "column": "C_CITY",
- "derived": null
- },
- {
- "name": "C_REGION",
- "table": "SSB.CUSTOMER",
- "column": "C_REGION",
- "derived": null
- },
- {
- "name": "C_NATION",
- "table": "SSB.CUSTOMER",
- "column": "C_NATION",
- "derived": null
- },
- {
- "name": "S_CITY",
- "table": "SSB.SUPPLIER",
- "column": "S_CITY",
- "derived": null
- },
- {
- "name": "S_REGION",
- "table": "SSB.SUPPLIER",
- "column": "S_REGION",
- "derived": null
- },
- {
- "name": "S_NATION",
- "table": "SSB.SUPPLIER",
- "column": "S_NATION",
- "derived": null
- },
- {
- "name": "D_YEAR",
- "table": "SSB.DATES",
- "column": "D_YEAR",
- "derived": null
- },
- {
- "name": "D_YEARMONTH",
- "table": "SSB.DATES",
- "column": "D_YEARMONTH",
- "derived": null
- },
- {
- "name": "D_YEARMONTHNUM",
- "table": "SSB.DATES",
- "column": "D_YEARMONTHNUM",
- "derived": null
- },
- {
- "name": "D_WEEKNUMINYEAR",
- "table": "SSB.DATES",
- "column": "D_WEEKNUMINYEAR",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "_COUNT_",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
+ "name" : "ssb",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "C_CITY",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_CITY",
+ "derived" : null
+ }, {
+ "name" : "C_REGION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_REGION",
+ "derived" : null
+ }, {
+ "name" : "C_NATION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_NATION",
+ "derived" : null
+ }, {
+ "name" : "S_CITY",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_CITY",
+ "derived" : null
+ }, {
+ "name" : "S_REGION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_REGION",
+ "derived" : null
+ }, {
+ "name" : "S_NATION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_REVENUE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "LO_REVENUE",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_SUPPLYCOST",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "LO_SUPPLYCOST",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_V_REVENUE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "V_REVENUE",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_V_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "V_REVENUE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "LO_PARTKEY",
- "encoding": "dict"
- },
- {
- "column": "C_CITY",
- "encoding": "dict"
- },
- {
- "column": "C_REGION",
- "encoding": "dict"
- },
- {
- "column": "C_NATION",
- "encoding": "dict"
- },
- {
- "column": "S_CITY",
- "encoding": "dict"
- },
- {
- "column": "S_REGION",
- "encoding": "dict"
- },
- {
- "column": "S_NATION",
- "encoding": "dict"
- },
- {
- "column": "D_YEAR",
- "encoding": "dict"
- },
- {
- "column": "D_YEARMONTH",
- "encoding": "dict"
- },
- {
- "column": "D_YEARMONTHNUM",
- "encoding": "dict"
- },
- {
- "column": "D_WEEKNUMINYEAR",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
},
- "signature": "5iV8LVYs+PmVUju8QNQ5TQ==",
- "last_modified": 1457503036686,
- "model_name": "ssb",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "_COUNT_",
- "TOTAL_REVENUE",
- "TOTAL_SUPPLYCOST",
- "TOTAL_V_REVENUE"
- ]
- }
- ]
- }
- ]
+ "signature" : "5iV8LVYs+PmVUju8QNQ5TQ==",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "LO_PARTKEY",
- "C_CITY",
- "C_REGION",
- "C_NATION",
- "S_CITY",
- "S_REGION",
- "S_NATION",
- "D_YEAR",
- "D_YEARMONTH",
- "D_YEARMONTHNUM",
- "D_WEEKNUMINYEAR"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "C_REGION",
- "C_NATION",
- "C_CITY"
- ],
- [
- "S_REGION",
- "S_NATION",
- "S_CITY"
- ],
- [
- "D_YEARMONTH",
- "D_YEARMONTHNUM",
- "D_WEEKNUMINYEAR"
- ]
- ],
- "mandatory_dims": [
- "D_YEAR"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ "D_YEAR" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": [],
- "status_need_notify": [],
- "partition_date_start": 694224000000,
- "partition_date_end": 3153600000000,
- "auto_merge_time_ranges": [
- 604800000,
- 2419200000
- ],
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
- "override_kylin_properties": {
- "kylin.hbase.default.compression.codec": "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid": "true"
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 694224000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.hbase.default.compression.codec" : "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index f62d196..4064fcb 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -1,245 +1,172 @@
{
- "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_with_slr_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- },
- {
- "name": "SELLER_ID",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "SELLER_ID",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+
+ "name" : "test_kylin_cube_with_slr_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "SELLER_ID",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "seller_id",
- "encoding": "int:4",
- "isShardBy": true
- },
- {
- "column": "cal_dt",
- "encoding": "dict"
- },
- {
- "column": "leaf_categ_id",
- "encoding": "fixed_length:18"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "fixed_length:12"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "seller_id",
+ "encoding" : "int:4",
+ "isShardBy" : true
+ }, {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "fixed_length:18"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801271,
- "model_name": "test_kylin_inner_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801271,
+ "model_name" : "test_kylin_inner_join_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name",
- "seller_id",
- "slr_segment_cd"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "seller_id"
- ],
- "joint_dims": [
- [
- "lstg_format_name",
- "lstg_site_id",
- "slr_segment_cd"
- ]
- ]
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : ["seller_id"],
+ "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
- "partition_date_start": 0
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
index e3a3e70..d4c64b5 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
@@ -1,249 +1,169 @@
{
- "uuid": "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_with_view_inner_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.V_TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_with_view_inner_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.V_TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_inner_join_view_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_inner_join_view_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ }]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
+ }
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
index b17fbff..0388c0e 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
@@ -1,249 +1,169 @@
{
- "uuid": "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_with_view_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.V_TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_with_view_left_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.V_TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_left_join_view_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_left_join_view_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ }]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
+ }
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index d185175..28328e4 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -1,4 +1,5 @@
{
+
"uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
"name": "test_kylin_cube_without_slr_desc",
"description": null,
@@ -159,19 +160,54 @@
"returntype": "extendedcolumn(100)"
},
"dependent_measure_ref": null
- },
- {
- "name": "PRICE_RAW",
- "function": {
- "expression": "RAW",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT",
+ "next_parameter" : null
},
- "returntype": "raw"
+ "returntype" : "raw"
},
- "dependent_measure_ref": null
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LEAF_CATEG_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LEAF_CATEG_ID",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
}
],
"rowkey": {
@@ -234,6 +270,9 @@
"item_count_sum",
"SITE_EXTENDED_1",
"SITE_EXTENDED_2",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 2aea1a8..ca1b35c 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -1,357 +1,293 @@
{
- "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_without_slr_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
+ "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_without_slr_left_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_CNT_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bitmap"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SITE_NAME_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SITE_NAME",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bitmap"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_FORMAT_CNT",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
},
- "dependent_measure_ref": null
+ "returntype" : "hllc(10)"
},
- {
- "name": "SELLER_CNT_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- },
- "returntype": "bitmap"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
},
- "dependent_measure_ref": null
+ "returntype" : "topn(100)",
+ "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
},
- {
- "name": "SITE_NAME_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SITE_NAME",
- "next_parameter": null
- },
- "returntype": "bitmap"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "hllc(10)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "topn(100)",
- "configuration": {
- "topn.encoding.SELLER_ID": "int:4"
- }
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LEAF_CATEG_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LEAF_CATEG_ID",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "PRICE_RAW",
- "function": {
- "expression": "RAW",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "raw"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "dictionaries": [
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "dictionaries" : [
{
- "column": "SITE_NAME",
+ "column" : "SITE_NAME",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
}
],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
- },
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum",
- "PRICE_RAW"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "seller_cnt_bitmap",
- "site_name_bitmap",
- "seller_format_cnt"
- ]
- }
- ]
- },
- {
- "name": "f3",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_left_join_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "seller_cnt_bitmap", "site_name_bitmap", "seller_format_cnt"]
+ } ]
+ }, {
+ "name" : "f3",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "top_seller" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"override_kylin_properties": {
"kylin.job.cubing.inmem.sampling.hll.precision": "16"
},
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index f2c4e72..ef10c1e 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -1,155 +1,118 @@
{
- "uuid": "901ed15e-7769-4c66-b7ae-fbdc971cd192",
- "name": "test_streaming_table_cube_desc",
- "description": "",
- "dimensions": [
- {
- "name": "DEFAULT.STREAMING_TABLE.SITE",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "SITE",
- "derived": null
- },
- {
- "name": "DEFAULT.STREAMING_TABLE.ITM",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "ITM",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "DAY_START",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "HOUR_START",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "MINUTE_START",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "_COUNT_",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd192",
+
+ "name" : "test_streaming_table_cube_desc",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "DEFAULT.STREAMING_TABLE.SITE",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "SITE",
+ "derived" : null
+ }, {
+ "name" : "DEFAULT.STREAMING_TABLE.ITM",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "ITM",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "DAY_START",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "HOUR_START",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "MINUTE_START",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "GMV",
- "next_parameter": null
- },
- "returntype": "decimal(19,6)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "GMV",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,6)"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "DAY_START",
- "encoding": "dict"
- },
- {
- "column": "HOUR_START",
- "encoding": "dict"
- },
- {
- "column": "MINUTE_START",
- "encoding": "dict"
- },
- {
- "column": "SITE",
- "encoding": "dict"
- },
- {
- "column": "ITM",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "DAY_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "HOUR_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "MINUTE_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "SITE",
+ "encoding" : "dict"
+ }, {
+ "column" : "ITM",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801314,
- "model_name": "test_streaming_table_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "_COUNT_",
- "GMV_SUM",
- "ITEM_COUNT_SUM"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801314,
+ "model_name" : "test_streaming_table_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "GMV_SUM", "ITEM_COUNT_SUM" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "DAY_START",
- "HOUR_START",
- "ITM",
- "MINUTE_START",
- "SITE"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "DAY_START",
- "HOUR_START",
- "MINUTE_START"
- ]
- ],
- "mandatory_dims": [],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "DAY_START", "HOUR_START", "ITM", "MINUTE_START", "SITE" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "DAY_START", "HOUR_START", "MINUTE_START" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
}
- ],
+ } ],
"override_kylin_properties": {
"kylin.cube.algorithm": "inmem"
},
- "notify_list": [],
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
\ No newline at end of file
[02/35] kylin git commit: KYLIN-1926-FK-PK-data-type-matching
Posted by li...@apache.org.
KYLIN-1926-FK-PK-data-type-matching
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ede67e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ede67e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ede67e3
Branch: refs/heads/1.5.x-HBase1.x
Commit: 4ede67e3f5a3bb1439d34263edb09c7ebf334675
Parents: eb92f96
Author: chenzhx <34...@qq.com>
Authored: Wed Sep 7 10:48:46 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 11:39:28 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeModel.js | 45 ++++++++++++--------
.../app/partials/modelDesigner/data_model.html | 5 ++-
2 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ede67e3/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
index be931f3..b20a829 100644
--- a/webapp/app/js/controllers/cubeModel.js
+++ b/webapp/app/js/controllers/cubeModel.js
@@ -59,7 +59,10 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
join: {
type: '',
primary_key: [],
- foreign_key: []
+ foreign_key: [],
+ isCompatible:[],
+ pk_type:[],
+ fk_type:[]
}
};
};
@@ -82,7 +85,6 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
backdrop: 'static',
scope: $scope
});
-
modalInstance.result.then(function () {
if (!$scope.lookupState.editing) {
$scope.doneAddLookup();
@@ -163,14 +165,37 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
};
+ $scope.changeKey = function(index){
+ var fact_table = modelsManager.selectedModel.fact_table;
+ var lookup_table = $scope.newLookup.table;
+ var pk_column = $scope.newLookup.join.primary_key[index];
+ var fk_column = $scope.newLookup.join.foreign_key[index];
+ if(pk_column!=='null'&&fk_column!=='null'){
+ $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(pk_column,lookup_table);
+ $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(fk_column,fact_table);
+ if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
+ $scope.newLookup.join.isCompatible[index]=false;
+ }else{
+ $scope.newLookup.join.isCompatible[index]=true;
+ }
+
+ }
+ }
+
$scope.addNewJoin = function(){
$scope.newLookup.join.primary_key.push("null");
$scope.newLookup.join.foreign_key.push("null");
+ $scope.newLookup.join.fk_type.push("null");
+ $scope.newLookup.join.pk_type.push("null");
+ $scope.newLookup.join.isCompatible.push(true);
};
$scope.removeJoin = function($index){
$scope.newLookup.join.primary_key.splice($index,1);
$scope.newLookup.join.foreign_key.splice($index,1);
+ $scope.newLookup.join.fk_type.splice($index,1);
+ $scope.newLookup.join.pk_type.splice($index,1);
+ $scope.newLookup.join.isCompatible.splice($index,1);
};
$scope.resetParams = function () {
@@ -195,22 +220,6 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
}
}
- //column type validate
- var fact_table = modelsManager.selectedModel.fact_table;
- var lookup_table = $scope.newLookup.table;
-
- for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
- var pk_column = $scope.newLookup.join.primary_key[i];
- var fk_column = $scope.newLookup.join.foreign_key[i];
- if(pk_column!=='null'&&fk_column!=='null'){
- var pk_type = TableModel.getColumnType(pk_column,lookup_table);
- var fk_type = TableModel.getColumnType(fk_column,fact_table);
- if(pk_type!==fk_type){
- errors.push(" Column Type incompatible "+pk_column+"["+pk_type+"]"+","+fk_column+"["+fk_type+"].");
- }
- }
- }
-
var errorInfo = "";
angular.forEach(errors,function(item){
errorInfo+="\n"+item;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ede67e3/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index d1a9cdd..662d0d9 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -157,13 +157,13 @@
<div ng-repeat="joinIndex in [] | range: newLookup.join.primary_key.length">
<div>
<select style="width: 45%" chosen data-placeholder="Fact Table Column"
- ng-model="newLookup.join.foreign_key[$index]"
+ ng-model="newLookup.join.foreign_key[$index]" ng-change="changeKey($index)"
ng-options="columns.name as columns.name for columns in getColumnsByTable(modelsManager.selectedModel.fact_table)" >
<option value=""></option>
</select>
<b>=</b>
<select style="width: 45%" chosen data-placeholder="Lookup Table Column"
- ng-model="newLookup.join.primary_key[$index]"
+ ng-model="newLookup.join.primary_key[$index]" ng-change="changeKey($index)"
ng-options="columns.name as columns.name for columns in getColumnsByTable(newLookup.table)" >
<option value=""></option>
</select>
@@ -173,6 +173,7 @@
</button>
</div>
<div class="space-4"></div>
+ <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}],{{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}].</b></small>
</div>
</div>
</div>
[26/35] kylin git commit: minor update hcatalog folder for emr
Posted by li...@apache.org.
minor update hcatalog folder for emr
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3450c0f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3450c0f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3450c0f9
Branch: refs/heads/1.5.x-HBase1.x
Commit: 3450c0f984933d40cc60a604813a310accdfaded
Parents: 56136ed
Author: shaofengshi <sh...@apache.org>
Authored: Sat Sep 10 17:52:56 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 18:00:08 2016 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3450c0f9/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 539a45b..71c2fe6 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -86,9 +86,9 @@ then
hcatalog_home=${hadoop_home}/hive/hcatalog
elif [ -d "${hive_home}/hcatalog" ]; then
hcatalog_home=${hive_home}/hcatalog
- elif [ -n is_aws ] && [ -d "/usr/lib/oozie/lib" ]; then
- # special handling for Amazon EMR, where hcat libs are under oozie!?
- hcatalog_home=/usr/lib/oozie/lib
+ elif [ -n is_aws ] && [ -d "/usr/lib/hive-hcatalog" ]; then
+ # special handling for Amazon EMR
+ hcatalog_home=/usr/lib/hive-hcatalog
else
echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
exit 1
[10/35] kylin git commit: KYLIN-2003 error start time
Posted by li...@apache.org.
KYLIN-2003 error start time
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ded3b585
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ded3b585
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ded3b585
Branch: refs/heads/1.5.x-HBase1.x
Commit: ded3b58583e76ab1234ac48459c264196c98f04b
Parents: d680169
Author: Jason <ji...@163.com>
Authored: Thu Sep 8 17:47:26 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu Sep 8 17:47:26 2016 +0800
----------------------------------------------------------------------
webapp/app/js/filters/filter.js | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ded3b585/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index 4e8d210..f9f7165 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -152,16 +152,16 @@ KylinApp
var convertedMillis = item;
if (gmttimezone.indexOf("GMT+") != -1) {
var offset = gmttimezone.substr(4, 1);
- convertedMillis = item + offset * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() + offset * 60 * 60000 + localOffset * 60000;
}
else if (gmttimezone.indexOf("GMT-") != -1) {
var offset = gmttimezone.substr(4, 1);
- convertedMillis = item - offset * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() - offset * 60 * 60000 + localOffset * 60000;
}
else {
// return PST by default
timezone = "PST";
- convertedMillis = item - 8 * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() - 8 * 60 * 60000 + localOffset * 60000;
}
return $filter('date')(convertedMillis, format) + " " + timezone;
[34/35] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 9b487a7..4a4f2a3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -25,7 +25,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -78,7 +79,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -91,7 +93,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -100,7 +102,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
admin.close();
@@ -109,8 +111,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -125,8 +126,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 7aecd7e..9dc9715 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -28,9 +28,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -99,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 4fe7748..65cf205 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -38,8 +38,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -64,7 +64,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -73,7 +73,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -132,7 +132,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 9adaf24..e1e2cba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
try {
CubeSegment cubeSegment = (CubeSegment) buildable;
- final HTableInterface hTable;
+ final Table hTable;
hTable = createHTable(cubeSegment);
List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements IStreamingOutput {
}
}
- private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException {
+ private Table createHTable(final CubeSegment cubeSegment) throws IOException {
final String hTableName = cubeSegment.getStorageLocationIdentifier();
CubeHTableUtil.createHTable(cubeSegment, null);
- final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+ final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!");
return hTable;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 3066fb5..45c6b91 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -86,7 +87,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -130,8 +131,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -229,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -322,8 +324,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -445,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -469,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -500,8 +501,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -535,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -558,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 295750a..efda4e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -60,7 +60,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -128,9 +128,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -187,10 +186,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index a1193e7..0aff276 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -40,7 +40,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -76,7 +77,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -159,10 +161,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -184,9 +186,9 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static void resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
logger.info("Unset coprocessor on " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -214,13 +216,13 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
}
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(Admin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processed = new ArrayList<String>();
for (String tableName : tableNames) {
@@ -331,7 +333,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index a5a85fa..29ca7b2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -26,10 +26,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 346c3a2..3ae411b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -30,12 +31,15 @@ import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -57,30 +61,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -104,7 +109,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- hBaseAdmin.close();
+ admin.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index f30f2c9..542df39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -31,15 +31,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -57,11 +57,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -118,8 +118,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -134,7 +134,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -169,8 +169,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -205,7 +205,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -223,8 +223,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 58ef7cb..63b5a42 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -21,12 +21,13 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -57,12 +58,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 4bd2c53..1681050 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -54,6 +56,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +75,8 @@ public class StorageCleanupJob extends AbstractApplication {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold();
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -150,22 +154,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 390930a..1196ed6 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -230,15 +231,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -307,6 +301,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -323,16 +322,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
[18/35] kylin git commit: minor, remove unnecessary raw measures
Posted by li...@apache.org.
minor, remove unnecessary raw measures
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/618cf28c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/618cf28c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/618cf28c
Branch: refs/heads/1.5.x-HBase1.x
Commit: 618cf28c96d6b267d13ff737d2b3c550fc67e176
Parents: a2c875d
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 18:57:01 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 18:57:01 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/topn/TopNMeasureType.java | 2 +
.../test_case_data/localmeta/cube_desc/ssb.json | 409 +++++++------
.../test_kylin_cube_with_slr_desc.json | 389 +++++++-----
...st_kylin_cube_with_view_inner_join_desc.json | 388 +++++++-----
...est_kylin_cube_with_view_left_join_desc.json | 388 +++++++-----
.../test_kylin_cube_without_slr_desc.json | 58 +-
...t_kylin_cube_without_slr_left_join_desc.json | 587 +++++++++++--------
.../test_streaming_table_cube_desc.json | 245 ++++----
8 files changed, 1423 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index ed22d61..0756056 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -274,6 +274,8 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (sum.isSum() == false)
return false;
+ if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0)
+ return false;
TblColRef sumCol = sum.getParameter().getColRefs().get(0);
return sumCol.equals(topnNumCol);
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index d3ea10b..4903979 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -1,179 +1,256 @@
{
- "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
- "name" : "ssb",
- "description" : "",
- "dimensions" : [ {
- "name" : "SSB.PART_DERIVED",
- "table" : "SSB.PART",
- "column" : null,
- "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
- }, {
- "name" : "C_CITY",
- "table" : "SSB.CUSTOMER",
- "column" : "C_CITY",
- "derived" : null
- }, {
- "name" : "C_REGION",
- "table" : "SSB.CUSTOMER",
- "column" : "C_REGION",
- "derived" : null
- }, {
- "name" : "C_NATION",
- "table" : "SSB.CUSTOMER",
- "column" : "C_NATION",
- "derived" : null
- }, {
- "name" : "S_CITY",
- "table" : "SSB.SUPPLIER",
- "column" : "S_CITY",
- "derived" : null
- }, {
- "name" : "S_REGION",
- "table" : "SSB.SUPPLIER",
- "column" : "S_REGION",
- "derived" : null
- }, {
- "name" : "S_NATION",
- "table" : "SSB.SUPPLIER",
- "column" : "S_NATION",
- "derived" : null
- }, {
- "name" : "D_YEAR",
- "table" : "SSB.DATES",
- "column" : "D_YEAR",
- "derived" : null
- }, {
- "name" : "D_YEARMONTH",
- "table" : "SSB.DATES",
- "column" : "D_YEARMONTH",
- "derived" : null
- }, {
- "name" : "D_YEARMONTHNUM",
- "table" : "SSB.DATES",
- "column" : "D_YEARMONTHNUM",
- "derived" : null
- }, {
- "name" : "D_WEEKNUMINYEAR",
- "table" : "SSB.DATES",
- "column" : "D_WEEKNUMINYEAR",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "_COUNT_",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ "uuid": "5c44df30-daec-486e-af90-927bf7851057",
+ "name": "ssb",
+ "description": "",
+ "dimensions": [
+ {
+ "name": "SSB.PART_DERIVED",
+ "table": "SSB.PART",
+ "column": null,
+ "derived": [
+ "P_MFGR",
+ "P_CATEGORY",
+ "P_BRAND"
+ ]
+ },
+ {
+ "name": "C_CITY",
+ "table": "SSB.CUSTOMER",
+ "column": "C_CITY",
+ "derived": null
+ },
+ {
+ "name": "C_REGION",
+ "table": "SSB.CUSTOMER",
+ "column": "C_REGION",
+ "derived": null
+ },
+ {
+ "name": "C_NATION",
+ "table": "SSB.CUSTOMER",
+ "column": "C_NATION",
+ "derived": null
+ },
+ {
+ "name": "S_CITY",
+ "table": "SSB.SUPPLIER",
+ "column": "S_CITY",
+ "derived": null
+ },
+ {
+ "name": "S_REGION",
+ "table": "SSB.SUPPLIER",
+ "column": "S_REGION",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_REVENUE",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "LO_REVENUE",
- "next_parameter" : null
+ {
+ "name": "S_NATION",
+ "table": "SSB.SUPPLIER",
+ "column": "S_NATION",
+ "derived": null
+ },
+ {
+ "name": "D_YEAR",
+ "table": "SSB.DATES",
+ "column": "D_YEAR",
+ "derived": null
+ },
+ {
+ "name": "D_YEARMONTH",
+ "table": "SSB.DATES",
+ "column": "D_YEARMONTH",
+ "derived": null
+ },
+ {
+ "name": "D_YEARMONTHNUM",
+ "table": "SSB.DATES",
+ "column": "D_YEARMONTHNUM",
+ "derived": null
+ },
+ {
+ "name": "D_WEEKNUMINYEAR",
+ "table": "SSB.DATES",
+ "column": "D_WEEKNUMINYEAR",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "_COUNT_",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_SUPPLYCOST",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "LO_SUPPLYCOST",
- "next_parameter" : null
+ {
+ "name": "TOTAL_REVENUE",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "LO_REVENUE",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_V_REVENUE",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "V_REVENUE",
- "next_parameter" : null
+ {
+ "name": "TOTAL_SUPPLYCOST",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "LO_SUPPLYCOST",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "LO_PARTKEY",
- "encoding" : "dict"
- }, {
- "column" : "C_CITY",
- "encoding" : "dict"
- }, {
- "column" : "C_REGION",
- "encoding" : "dict"
- }, {
- "column" : "C_NATION",
- "encoding" : "dict"
- }, {
- "column" : "S_CITY",
- "encoding" : "dict"
- }, {
- "column" : "S_REGION",
- "encoding" : "dict"
- }, {
- "column" : "S_NATION",
- "encoding" : "dict"
- }, {
- "column" : "D_YEAR",
- "encoding" : "dict"
- }, {
- "column" : "D_YEARMONTH",
- "encoding" : "dict"
- }, {
- "column" : "D_YEARMONTHNUM",
- "encoding" : "dict"
- }, {
- "column" : "D_WEEKNUMINYEAR",
- "encoding" : "dict"
- } ]
+ {
+ "name": "TOTAL_V_REVENUE",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "V_REVENUE",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "LO_PARTKEY",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_CITY",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_REGION",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_NATION",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_CITY",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_REGION",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_NATION",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEAR",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEARMONTH",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEARMONTHNUM",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_WEEKNUMINYEAR",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : "5iV8LVYs+PmVUju8QNQ5TQ==",
- "last_modified" : 1457503036686,
- "model_name" : "ssb",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
- } ]
- } ]
+ "signature": "5iV8LVYs+PmVUju8QNQ5TQ==",
+ "last_modified": 1457503036686,
+ "model_name": "ssb",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "_COUNT_",
+ "TOTAL_REVENUE",
+ "TOTAL_SUPPLYCOST",
+ "TOTAL_V_REVENUE"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
- "mandatory_dims" : [ "D_YEAR" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "LO_PARTKEY",
+ "C_CITY",
+ "C_REGION",
+ "C_NATION",
+ "S_CITY",
+ "S_REGION",
+ "S_NATION",
+ "D_YEAR",
+ "D_YEARMONTH",
+ "D_YEARMONTHNUM",
+ "D_WEEKNUMINYEAR"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "C_REGION",
+ "C_NATION",
+ "C_CITY"
+ ],
+ [
+ "S_REGION",
+ "S_NATION",
+ "S_CITY"
+ ],
+ [
+ "D_YEARMONTH",
+ "D_YEARMONTHNUM",
+ "D_WEEKNUMINYEAR"
+ ]
+ ],
+ "mandatory_dims": [
+ "D_YEAR"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : [ ],
- "status_need_notify" : [ ],
- "partition_date_start" : 694224000000,
- "partition_date_end" : 3153600000000,
- "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ ],
+ "notify_list": [],
+ "status_need_notify": [],
+ "partition_date_start": 694224000000,
+ "partition_date_end": 3153600000000,
+ "auto_merge_time_ranges": [
+ 604800000,
+ 2419200000
+ ],
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
+ "override_kylin_properties": {
+ "kylin.hbase.default.compression.codec": "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid": "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 4064fcb..f62d196 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -1,172 +1,245 @@
{
- "uuid" : "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
-
- "name" : "test_kylin_cube_with_slr_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- }, {
- "name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+ "name": "test_kylin_cube_with_slr_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ },
+ {
+ "name": "SELLER_ID",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "SELLER_ID",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "seller_id",
- "encoding" : "int:4",
- "isShardBy" : true
- }, {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "fixed_length:18"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "fixed_length:12"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "seller_id",
+ "encoding": "int:4",
+ "isShardBy": true
+ },
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "fixed_length:18"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "fixed_length:12"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801271,
- "model_name" : "test_kylin_inner_join_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801271,
+ "model_name": "test_kylin_inner_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : ["seller_id"],
- "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name",
+ "seller_id",
+ "slr_segment_cd"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "seller_id"
+ ],
+ "joint_dims": [
+ [
+ "lstg_format_name",
+ "lstg_site_id",
+ "slr_segment_cd"
+ ]
+ ]
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "partition_date_start" : 0
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
+ "partition_date_start": 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
index d4c64b5..e3a3e70 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
@@ -1,169 +1,249 @@
{
- "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_inner_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_with_view_inner_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.V_TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_inner_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_inner_join_view_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
index 0388c0e..b17fbff 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
@@ -1,169 +1,249 @@
{
- "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_left_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_with_view_left_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.V_TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_left_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_left_join_view_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 28328e4..7de2ae2 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
"name": "test_kylin_cube_without_slr_desc",
"description": null,
@@ -160,54 +159,19 @@
"returntype": "extendedcolumn(100)"
},
"dependent_measure_ref": null
- }, {
- "name" : "CAL_DT_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "CAL_DT",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "LSTG_FORMAT_NAME_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "LEAF_CATEG_ID_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LEAF_CATEG_ID",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "PRICE_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ },
+ {
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
},
- "returntype" : "raw"
+ "returntype": "raw"
},
- "dependent_measure_ref" : null
+ "dependent_measure_ref": null
}
],
"rowkey": {
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index ca1b35c..4270aab 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -1,293 +1,360 @@
{
- "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_without_slr_left_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_without_slr_left_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+ },
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_CNT_BITMAP",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bitmap"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SITE_NAME_BITMAP",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SITE_NAME",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bitmap"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "hllc(10)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOP_SELLER",
- "function" : {
- "expression" : "TOP_N",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "topn(100)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "CAL_DT_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "CAL_DT",
- "next_parameter" : null
+ {
+ "name": "SELLER_CNT_BITMAP",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ },
+ "returntype": "bitmap"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "LSTG_FORMAT_NAME_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
+ {
+ "name": "SITE_NAME_BITMAP",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "SITE_NAME",
+ "next_parameter": null
+ },
+ "returntype": "bitmap"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "LEAF_CATEG_ID_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LEAF_CATEG_ID",
- "next_parameter" : null
+ {
+ "name": "SELLER_FORMAT_CNT",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "LSTG_FORMAT_NAME",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ }
+ },
+ "returntype": "hllc(10)"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "PRICE_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ }
+ },
+ "returntype": "topn(100)",
+ "configuration": {
+ "topn.encoding.SELLER_ID": "int:4"
+ }
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "dictionaries" : [
{
- "column" : "SITE_NAME",
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "dictionaries": [
+ {
+ "column": "SITE_NAME",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
}
],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_left_join_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ]
- } ]
- }, {
- "name" : "f2",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "seller_cnt_bitmap", "site_name_bitmap", "seller_format_cnt"]
- } ]
- }, {
- "name" : "f3",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "top_seller" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_left_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
+ "PRICE_RAW"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "f2",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "seller_cnt_bitmap",
+ "site_name_bitmap",
+ "seller_format_cnt"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "f3",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "top_seller"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"override_kylin_properties": {
"kylin.job.cubing.inmem.sampling.hll.precision": "16"
},
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index ef10c1e..f2c4e72 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -1,118 +1,155 @@
{
- "uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd192",
-
- "name" : "test_streaming_table_cube_desc",
- "description" : "",
- "dimensions" : [ {
- "name" : "DEFAULT.STREAMING_TABLE.SITE",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "SITE",
- "derived" : null
- }, {
- "name" : "DEFAULT.STREAMING_TABLE.ITM",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "ITM",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "DAY_START",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "HOUR_START",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "MINUTE_START",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "_COUNT_",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ "uuid": "901ed15e-7769-4c66-b7ae-fbdc971cd192",
+ "name": "test_streaming_table_cube_desc",
+ "description": "",
+ "dimensions": [
+ {
+ "name": "DEFAULT.STREAMING_TABLE.SITE",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "SITE",
+ "derived": null
+ },
+ {
+ "name": "DEFAULT.STREAMING_TABLE.ITM",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "ITM",
+ "derived": null
+ },
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "DAY_START",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "GMV",
- "next_parameter" : null
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "HOUR_START",
+ "derived": null
+ },
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "MINUTE_START",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "_COUNT_",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "decimal(19,6)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "GMV",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,6)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "DAY_START",
- "encoding" : "dict"
- }, {
- "column" : "HOUR_START",
- "encoding" : "dict"
- }, {
- "column" : "MINUTE_START",
- "encoding" : "dict"
- }, {
- "column" : "SITE",
- "encoding" : "dict"
- }, {
- "column" : "ITM",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "DAY_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "HOUR_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "MINUTE_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "SITE",
+ "encoding": "dict"
+ },
+ {
+ "column": "ITM",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801314,
- "model_name" : "test_streaming_table_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "_COUNT_", "GMV_SUM", "ITEM_COUNT_SUM" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801314,
+ "model_name": "test_streaming_table_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "_COUNT_",
+ "GMV_SUM",
+ "ITEM_COUNT_SUM"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "DAY_START", "HOUR_START", "ITM", "MINUTE_START", "SITE" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "DAY_START", "HOUR_START", "MINUTE_START" ] ],
- "mandatory_dims" : [ ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "DAY_START",
+ "HOUR_START",
+ "ITM",
+ "MINUTE_START",
+ "SITE"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "DAY_START",
+ "HOUR_START",
+ "MINUTE_START"
+ ]
+ ],
+ "mandatory_dims": [],
+ "joint_dims": []
+ }
}
- } ],
+ ],
"override_kylin_properties": {
"kylin.cube.algorithm": "inmem"
},
- "notify_list" : [ ],
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ "notify_list": [],
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
\ No newline at end of file
[35/35] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d69da92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d69da92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d69da92
Branch: refs/heads/1.5.x-HBase1.x
Commit: 6d69da921a8c766a70c880a59aa9afe82400664d
Parents: 6db4b17
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Sep 12 19:17:15 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../test_case_data/sandbox/kylin_job_conf.xml | 86 +++++++++---------
examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
.../kylin/provision/BuildCubeWithEngine.java | 13 ++-
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 36 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../kylin/storage/hbase/HBaseStorage.java | 3 +-
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +-
.../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 14 ++-
.../observer/ObserverAggregationCache.java | 10 ++-
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../hbase/steps/HBaseStreamingOutput.java | 9 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 22 ++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 26 ++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
51 files changed, 459 insertions(+), 520 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index bd947af..6082fa9 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -1,20 +1,18 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
-->
+
<configuration>
<property>
@@ -26,44 +24,41 @@
</description>
</property>
- <property>
- <name>mapreduce.map.maxattempts</name>
- <value>2</value>
- </property>
+ <!-- uncomment the following 5 properties to enable lzo compressing
- <!--
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
- <property>
- <name>mapred.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
+ <property>
+ <name>mapred.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
--->
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ !-->
<property>
<name>mapreduce.job.max.split.locations</name>
@@ -76,5 +71,4 @@
<value>2</value>
<description>Block replication</description>
</property>
-
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 18f6feb..ff1c7eb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -28,12 +28,12 @@
<property>
<name>mapred.job.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
<name>mapred.job.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -58,7 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,10 +81,9 @@
<value>false</value>
</property>
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
+ <value>0.05</value>
</property>
<property>
@@ -114,7 +113,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -124,7 +123,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -169,7 +168,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -219,7 +218,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -234,7 +233,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -244,7 +243,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 3d60a3c..0910df5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -55,6 +54,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
@@ -419,10 +419,10 @@ public class BuildCubeWithEngine {
}
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
@@ -448,5 +448,4 @@ public class BuildCubeWithEngine {
}
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 38f9365..3cad925 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.8.1</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 38f299e..bfb5fe4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -37,6 +37,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index ab18029..d55edc3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -57,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4cd527c..945a111 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,8 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -424,35 +423,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
-
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- if (null != table) {
- table.close();
- }
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index df296cf..340f142 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -45,11 +45,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
@@ -137,14 +137,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -170,14 +169,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -189,12 +187,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..50524b6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- private HConnection getConnection() throws IOException {
+ private Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index f4dfd2b..3d82105 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
@@ -36,6 +35,8 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 8ac3832..982a044 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,11 +25,12 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index ff729f4..1944327 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -46,10 +46,10 @@ import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -152,7 +152,7 @@ public class CubeStorageQuery implements IStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 6342c5c..9a171e9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +51,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +61,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +95,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7139ca7..7e25e4c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index a77f988..49a0f1c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -116,8 +117,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +127,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +161,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..331e34d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +113,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +123,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +162,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index 394b3e2..9fd33f5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,9 +23,9 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 573951b..fbb2cf9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -48,10 +49,10 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,7 +115,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -163,7 +164,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index f1e5dab..68c9534 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -164,8 +165,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d69da92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 36adca1..0073e07 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -145,7 +145,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -181,7 +181,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
[24/35] kylin git commit: KYLIN-2004 Make the creating intermediate
hive table steps configurable
Posted by li...@apache.org.
KYLIN-2004 Make the creating intermediate hive table steps configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/233a699f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/233a699f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/233a699f
Branch: refs/heads/1.5.x-HBase1.x
Commit: 233a699f3b6f7a6c64ecf43fb80108b56db61f5f
Parents: d7cbf67
Author: shaofengshi <sh...@apache.org>
Authored: Fri Sep 9 19:04:10 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 17:59:46 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 48 ++++--
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/job/execution/AbstractExecutable.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 2 +-
.../kylin/metadata/model/DataModelDesc.java | 8 +-
...t_kylin_cube_without_slr_left_join_desc.json | 3 +-
.../kylin/rest/controller/CubeController.java | 2 +-
.../source/hive/CreateFlatHiveTableStep.java | 32 +++-
.../apache/kylin/source/hive/HiveMRInput.java | 169 ++++++++++++++++++-
10 files changed, 234 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ac9d48..de9051c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -805,7 +805,7 @@ abstract public class KylinConfigBase implements Serializable {
setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize));
}
- public boolean getTableJoinTypeCheck() {
- return Boolean.valueOf(this.getOptional("kylin.table.join.strong.check", "true"));
+ public String getCreateFlatHiveTableMethod() {
+ return getOptional("kylin.hive.create.flat.table.method", "1");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index b39265d..699d084 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -107,14 +107,14 @@ public class JoinedFlatTable {
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) {
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig, boolean redistribute) {
StringBuilder sql = new StringBuilder();
sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
+ sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc, redistribute) + ";").append("\n");
return sql.toString();
}
- public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
+ public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc, boolean redistribute) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT" + "\n");
String tableAlias;
@@ -129,7 +129,15 @@ public class JoinedFlatTable {
}
appendJoinStatement(flatDesc, sql, tableAliasMap);
appendWhereStatement(flatDesc, sql, tableAliasMap);
- appendDistributeStatement(flatDesc, sql, tableAliasMap);
+ if (redistribute == true) {
+ String redistributeCol = null;
+ TblColRef distDcol = flatDesc.getDistributedBy();
+ if (distDcol != null) {
+ String tblAlias = tableAliasMap.get(distDcol.getTable());
+ redistributeCol = tblAlias + "." + distDcol.getName();
+ }
+ appendDistributeStatement(sql, redistributeCol);
+ }
return sql.toString();
}
@@ -228,14 +236,11 @@ public class JoinedFlatTable {
return result;
}
- private static void appendDistributeStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
- TblColRef distDcol = flatDesc.getDistributedBy();
-
- if (distDcol != null) {
- String tblAlias = tableAliasMap.get(distDcol.getTable());
- sql.append(" DISTRIBUTE BY ").append(tblAlias).append(".").append(distDcol.getName());
+ private static void appendDistributeStatement(StringBuilder sql, String redistributeCol) {
+ if (redistributeCol != null) {
+ sql.append(" DISTRIBUTE BY ").append(redistributeCol).append(";\n");
} else {
- sql.append(" DISTRIBUTE BY RAND()");
+ sql.append(" DISTRIBUTE BY RAND()").append(";\n");
}
}
@@ -280,4 +285,25 @@ public class JoinedFlatTable {
return hiveDataType.toLowerCase();
}
+ public static String generateSelectRowCountStatement(IJoinedFlatTableDesc intermediateTableDesc, String outputDir) {
+ StringBuilder sql = new StringBuilder();
+ sql.append("set hive.exec.compress.output=false;\n");
+ sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + intermediateTableDesc.getTableName() + ";\n");
+ return sql.toString();
+ }
+
+ public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc intermediateTableDesc) {
+ final String tableName = intermediateTableDesc.getTableName();
+ StringBuilder sql = new StringBuilder();
+ sql.append("INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName);
+
+ String redistributeCol = null;
+ TblColRef distDcol = intermediateTableDesc.getDistributedBy();
+ if (distDcol != null) {
+ redistributeCol = colName(distDcol.getCanonicalName());
+ }
+ appendDistributeStatement(sql, redistributeCol);
+ return sql.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 6084e7b..893c034 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -56,5 +56,6 @@ public final class ExecutableConstants {
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
public static final String STEP_NAME_UPDATE_II_INFO = "Update Inverted Index Info";
+ public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table";
public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 4dedad1..09f9b54 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -49,7 +49,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected static final String START_TIME = "startTime";
protected static final String END_TIME = "endTime";
- private static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
+ protected static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
protected int retry = 0;
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
index 0faf22a..1fe47f8 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
@@ -77,7 +77,7 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
@Test
public void testGenerateInsertSql() throws IOException {
- String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()), true);
System.out.println(sqls);
int length = sqls.length();
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7f5edfe..d04830b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -314,13 +314,7 @@ public class DataModelDesc extends RootPersistentEntity {
}
for (int i = 0; i < fkCols.length; i++) {
if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) {
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final String msg = "Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype();
- if (kylinConfig.getTableJoinTypeCheck() == true) {
- throw new IllegalStateException(msg);
- } else {
- logger.warn(msg);
- }
+ logger.warn("Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index ca1b35c..0470dc6 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -289,7 +289,8 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties": {
- "kylin.job.cubing.inmem.sampling.hll.precision": "16"
+ "kylin.job.cubing.inmem.sampling.hll.precision": "16",
+ "kylin.hive.create.flat.table.method": "2"
},
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 7081d02..5397df7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -152,7 +152,7 @@ public class CubeController extends BasicController {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
- String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc);
+ String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
GeneralResponse repsonse = new GeneralResponse();
repsonse.setProperty("sql", sql);
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index cd32f9c..bcb9a38 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -76,8 +76,11 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
private void createFlatHiveTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(getInitStatement());
- hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
- hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); //disable merge
+ boolean useRedistribute = getUseRedistribute();
+ if (useRedistribute == true) {
+ hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
+ hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); //disable merge
+ }
hiveCmdBuilder.addStatement(getCreateTableStatement());
final String cmd = hiveCmdBuilder.toString();
@@ -101,13 +104,20 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig config = getCubeSpecificConfig();
try {
- long rowCount = readRowCountFromFile();
- if (!config.isEmptySegmentAllowed() && rowCount == 0) {
- stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
- return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+
+ boolean useRedistribute = getUseRedistribute();
+
+ int numReducers = 0;
+ if (useRedistribute == true) {
+ long rowCount = readRowCountFromFile();
+ if (!config.isEmptySegmentAllowed() && rowCount == 0) {
+ stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+
+ numReducers = determineNumReducer(config, rowCount);
}
- int numReducers = determineNumReducer(config, rowCount);
createFlatHiveTable(config, numReducers);
return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
@@ -125,6 +135,14 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
return getParam("HiveInit");
}
+ public void setUseRedistribute(boolean useRedistribute) {
+ setParam("useRedistribute", String.valueOf(useRedistribute));
+ }
+
+ public boolean getUseRedistribute() {
+ return Boolean.valueOf(getParam("useRedistribute"));
+ }
+
public void setCreateTableStatement(String sql) {
setParam("HiveRedistributeData", sql);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e3d7879..3ea9af5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,8 +19,10 @@
package org.apache.kylin.source.hive;
import java.io.IOException;
+import java.io.InputStream;
import java.util.Set;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -28,6 +30,11 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BufferedLogger;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -110,16 +117,46 @@ public class HiveMRInput implements IMRInput {
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
- final String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()) + "/row_count";
+ final KylinConfig kylinConfig = CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
+
+ String createFlatTableMethod = kylinConfig.getCreateFlatHiveTableMethod();
+ if ("1".equals(createFlatTableMethod)) {
+ // create flat table first, then count and redistribute
+ jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, false, ""));
+ jobFlow.addTask(createRedistributeFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName));
+ } else if ("2".equals(createFlatTableMethod)) {
+ // count from source table first, and then redistribute, suitable for partitioned table
+ final String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()) + "/row_count";
+ jobFlow.addTask(createCountHiveTableStep(conf, flatDesc, jobFlow.getId(), rowCountOutputDir));
+ jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, true, rowCountOutputDir));
+ } else {
+ throw new IllegalArgumentException("Unknown value for kylin.hive.create.flat.table.method: " + createFlatTableMethod);
+ }
- jobFlow.addTask(createCountHiveTableStep(conf, flatDesc, jobFlow.getId(), rowCountOutputDir));
- jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, rowCountOutputDir));
AbstractExecutable task = createLookupHiveViewMaterializationStep(jobFlow.getId());
if (task != null) {
jobFlow.addTask(task);
}
}
+ public static AbstractExecutable createRedistributeFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName) {
+ StringBuilder hiveInitBuf = new StringBuilder();
+ hiveInitBuf.append("USE ").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
+ hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
+
+ String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
+
+ RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
+ step.setInitStatement(hiveInitBuf.toString());
+ step.setSelectRowCountStatement(JoinedFlatTable.generateSelectRowCountStatement(flatTableDesc, rowCountOutputDir));
+ step.setRowCountOutputDir(rowCountOutputDir);
+ step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatTableDesc));
+ CubingExecutableUtil.setCubeName(cubeName, step.getParams());
+ step.setName(ExecutableConstants.STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE);
+ return step;
+ }
+
+
public static AbstractExecutable createCountHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String rowCountOutputDir) {
final ShellExecutable step = new ShellExecutable();
@@ -174,17 +211,17 @@ public class HiveMRInput implements IMRInput {
return step;
}
- public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, String rowCountOutputDir) {
+ public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, boolean redistribute, String rowCountOutputDir) {
StringBuilder hiveInitBuf = new StringBuilder();
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
- String insertDataHqls;
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf);
+ String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf, redistribute);
CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
+ step.setUseRedistribute(redistribute);
step.setInitStatement(hiveInitBuf.toString());
step.setRowCountOutputDir(rowCountOutputDir);
step.setCreateTableStatement(useDatabaseHql + dropTableHql + createTableHql + insertDataHqls);
@@ -213,6 +250,126 @@ public class HiveMRInput implements IMRInput {
}
}
+ public static class RedistributeFlatHiveTableStep extends AbstractExecutable {
+ private final BufferedLogger stepLogger = new BufferedLogger(logger);
+
+ private void computeRowCount(CliCommandExecutor cmdExecutor) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(getInitStatement());
+ hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n");
+ hiveCmdBuilder.addStatement(getSelectRowCountStatement());
+ final String cmd = hiveCmdBuilder.build();
+
+ stepLogger.log("Compute row count of flat hive table, cmd: ");
+ stepLogger.log(cmd);
+
+ Pair<Integer, String> response = cmdExecutor.execute(cmd, stepLogger);
+ if (response.getFirst() != 0) {
+ throw new RuntimeException("Failed to compute row count of flat hive table");
+ }
+ }
+
+ private long readRowCountFromFile(Path file) throws IOException {
+ FileSystem fs = FileSystem.get(file.toUri(), HadoopUtil.getCurrentConfiguration());
+ InputStream in = fs.open(file);
+ try {
+ String content = IOUtils.toString(in);
+ return Long.valueOf(content.trim()); // strip the '\n' character
+
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
+ }
+
+ private int determineNumReducer(KylinConfig config) throws IOException {
+ computeRowCount(config.getCliCommandExecutor());
+
+ Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
+ long rowCount = readRowCountFromFile(rowCountFile);
+ int mapperInputRows = config.getHadoopJobMapperInputRows();
+
+ int numReducers = Math.round(rowCount / ((float) mapperInputRows));
+ numReducers = Math.max(1, numReducers);
+
+ stepLogger.log("total input rows = " + rowCount);
+ stepLogger.log("expected input rows per mapper = " + mapperInputRows);
+ stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
+
+ return numReducers;
+ }
+
+ private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(getInitStatement());
+ hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
+ hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n");
+ hiveCmdBuilder.addStatement(getRedistributeDataStatement());
+ final String cmd = hiveCmdBuilder.toString();
+
+ stepLogger.log("Redistribute table, cmd: ");
+ stepLogger.log(cmd);
+
+ Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger);
+ if (response.getFirst() != 0) {
+ throw new RuntimeException("Failed to redistribute flat hive table");
+ }
+ }
+
+ private KylinConfig getCubeSpecificConfig() {
+ String cubeName = CubingExecutableUtil.getCubeName(getParams());
+ CubeManager manager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube = manager.getCube(cubeName);
+ return cube.getConfig();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ KylinConfig config = getCubeSpecificConfig();
+
+ try {
+ int numReducers = determineNumReducer(config);
+ redistributeTable(config, numReducers);
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
+
+ } catch (Exception e) {
+ logger.error("job:" + getId() + " execute finished with exception", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+ }
+
+ public void setInitStatement(String sql) {
+ setParam("HiveInit", sql);
+ }
+
+ public String getInitStatement() {
+ return getParam("HiveInit");
+ }
+
+ public void setSelectRowCountStatement(String sql) {
+ setParam("HiveSelectRowCount", sql);
+ }
+
+ public String getSelectRowCountStatement() {
+ return getParam("HiveSelectRowCount");
+ }
+
+ public void setRedistributeDataStatement(String sql) {
+ setParam("HiveRedistributeData", sql);
+ }
+
+ public String getRedistributeDataStatement() {
+ return getParam("HiveRedistributeData");
+ }
+
+ public void setRowCountOutputDir(String rowCountOutputDir) {
+ setParam("rowCountOutputDir", rowCountOutputDir);
+ }
+
+ public String getRowCountOutputDir() {
+ return getParam("rowCountOutputDir");
+ }
+ }
+
public static class GarbageCollectionStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(GarbageCollectionStep.class);
[09/35] kylin git commit: minor, better logging messages
Posted by li...@apache.org.
minor, better logging messages
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6801695
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6801695
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6801695
Branch: refs/heads/1.5.x-HBase1.x
Commit: d6801695240f5191716aec4cbd336470281fb20f
Parents: 01d033f
Author: Li Yang <li...@apache.org>
Authored: Thu Sep 8 17:02:17 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Sep 8 17:02:28 2016 +0800
----------------------------------------------------------------------
build/bin/setenv.sh | 2 +-
.../java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java | 2 +-
.../java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java | 2 +-
.../java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java | 4 ++--
.../org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/build/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
index d486672..317005b 100755
--- a/build/bin/setenv.sh
+++ b/build/bin/setenv.sh
@@ -20,7 +20,7 @@
# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
# uncomment following to for it to take effect
export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss256K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
# uncomment following to for it to take effect(the values need adjusting to fit your env)
# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
index 33b6f29..0b4ae40 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
@@ -56,7 +56,7 @@ public class BatchMergeJobBuilder extends JobBuilderSupport {
final String cuboidRootPath = getCuboidRootPath(jobId);
final List<CubeSegment> mergingSegments = cubeSegment.getCubeInstance().getMergingSegments(cubeSegment);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + cubeSegment);
final List<String> mergingSegmentIds = Lists.newArrayList();
final List<String> mergingCuboidPaths = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index 289cd48..129d525 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -48,7 +48,7 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
final String jobId = result.getId();
final List<CubeSegment> mergingSegments = cubeSegment.getCubeInstance().getMergingSegments(cubeSegment);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + cubeSegment);
final List<String> mergingSegmentIds = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index 1bd052d..7c2b3fd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -169,7 +169,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
public List<String> getMergingHTables() {
final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + seg);
final List<String> mergingHTables = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingHTables.add(merging.getStorageLocationIdentifier());
@@ -179,7 +179,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
public List<String> getMergingHDFSPaths() {
final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + seg);
final List<String> mergingHDFSPaths = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingHDFSPaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index 729635b..bdd3981 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -73,7 +73,7 @@ public class ZookeeperJobLock implements JobLock {
logger.warn("error acquire lock", e);
}
if (!hasLock) {
- logger.warn("fail to acquire lock, scheduler has not been started");
+ logger.warn("fail to acquire lock, scheduler has not been started; maybe another kylin process is still running?");
zkClient.close();
return false;
}
[17/35] kylin git commit: KYLIN-2005 Move all storage side behavior
hints to GTScanRequest
Posted by li...@apache.org.
KYLIN-2005 Move all storage side behavior hints to GTScanRequest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2c875d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2c875d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2c875d8
Branch: refs/heads/1.5.x-HBase1.x
Commit: a2c875d8a2d06f23dd6467bbcc459bff82918295
Parents: e38557b
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 16:46:22 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 17:47:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/gridtable/GTScanRequest.java | 33 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 30 +-
.../kylin/gridtable/StorageSideBehavior.java | 30 +
.../apache/kylin/query/ITKylinQueryTest.java | 4 +-
.../common/coprocessor/CoprocessorBehavior.java | 30 -
.../observer/AggregateRegionObserver.java | 10 +-
.../observer/AggregationScanner.java | 16 +-
.../coprocessor/observer/ObserverEnabler.java | 6 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 88 +--
.../hbase/cube/v2/ExpectedSizeIterator.java | 4 +-
.../coprocessor/endpoint/CubeVisitService.java | 18 +-
.../endpoint/generated/CubeVisitProtos.java | 754 ++++---------------
.../endpoint/protobuf/CubeVisit.proto | 13 +-
.../observer/AggregateRegionObserverTest.java | 6 +-
14 files changed, 332 insertions(+), 710 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5d27028..3e57e86 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -59,6 +59,9 @@ public class GTScanRequest {
private String[] aggrMetricsFuncs;//
// hint to storage behavior
+ private String storageBehavior;
+ private long startTime;
+ private long timeout;
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
private int storageScanRowNumThreshold;
@@ -69,7 +72,7 @@ public class GTScanRequest {
GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, boolean allowStorageAggregation, //
- double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit) {
+ double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit, String storageBehavior, long startTime, long timeout) {
this.info = info;
if (ranges == null) {
this.ranges = Lists.newArrayList(new GTScanRange(new GTRecord(info), new GTRecord(info)));
@@ -83,6 +86,9 @@ public class GTScanRequest {
this.aggrMetrics = aggrMetrics;
this.aggrMetricsFuncs = aggrMetricsFuncs;
+ this.storageBehavior = storageBehavior;
+ this.startTime = startTime;
+ this.timeout = timeout;
this.allowStorageAggregation = allowStorageAggregation;
this.aggCacheMemThreshold = aggCacheMemThreshold;
this.storageScanRowNumThreshold = storageScanRowNumThreshold;
@@ -115,6 +121,10 @@ public class GTScanRequest {
}
}
+ public void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
private void validateFilterPushDown(GTInfo info) {
if (!hasFilterPushDown())
return;
@@ -280,6 +290,18 @@ public class GTScanRequest {
return this.storagePushDownLimit;
}
+ public String getStorageBehavior() {
+ return storageBehavior;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
@@ -320,6 +342,9 @@ public class GTScanRequest {
out.putDouble(value.aggCacheMemThreshold);
BytesUtil.writeVInt(value.storageScanRowNumThreshold, out);
BytesUtil.writeVInt(value.storagePushDownLimit, out);
+ BytesUtil.writeVLong(value.startTime, out);
+ BytesUtil.writeVLong(value.timeout, out);
+ BytesUtil.writeUTFString(value.storageBehavior, out);
}
@Override
@@ -350,11 +375,15 @@ public class GTScanRequest {
double sAggrCacheGB = in.getDouble();
int storageScanRowNumThreshold = BytesUtil.readVInt(in);
int storagePushDownLimit = BytesUtil.readVInt(in);
+ long startTime = BytesUtil.readVLong(in);
+ long timeout = BytesUtil.readVLong(in);
+ String storageBehavior = BytesUtil.readUTFString(in);
return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns).//
setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs).//
setFilterPushDown(sGTFilter).setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB).//
- setStorageScanRowNumThreshold(storageScanRowNumThreshold).setStoragePushDownLimit(storagePushDownLimit).createGTScanRequest();
+ setStorageScanRowNumThreshold(storageScanRowNumThreshold).setStoragePushDownLimit(storagePushDownLimit).//
+ setStartTime(startTime).setTimeout(timeout).setStorageBehavior(storageBehavior).createGTScanRequest();
}
private void serializeGTRecord(GTRecord gtRecord, ByteBuffer out) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index c4390cd..f542de1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -21,6 +21,7 @@ package org.apache.kylin.gridtable;
import java.util.BitSet;
import java.util.List;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -36,6 +37,9 @@ public class GTScanRequestBuilder {
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.
private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced.
+ private long startTime = -1;
+ private long timeout = -1;
+ private String storageBehavior = null;
public GTScanRequestBuilder setInfo(GTInfo info) {
this.info = info;
@@ -92,6 +96,21 @@ public class GTScanRequestBuilder {
return this;
}
+ public GTScanRequestBuilder setStartTime(long startTime) {
+ this.startTime = startTime;
+ return this;
+ }
+
+ public GTScanRequestBuilder setTimeout(long timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ public GTScanRequestBuilder setStorageBehavior(String storageBehavior) {
+ this.storageBehavior = storageBehavior;
+ return this;
+ }
+
public GTScanRequest createGTScanRequest() {
if (aggrGroupBy == null) {
aggrGroupBy = new ImmutableBitSet(new BitSet());
@@ -104,7 +123,14 @@ public class GTScanRequestBuilder {
if (aggrMetricsFuncs == null) {
aggrMetricsFuncs = new String[0];
}
-
- return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit);
+
+ if (storageBehavior == null) {
+ storageBehavior = BackdoorToggles.getCoprocessorBehavior() == null ? StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
+ }
+
+ this.startTime = startTime == -1 ? System.currentTimeMillis() : startTime;
+ this.timeout = timeout == -1 ? 300000 : timeout;
+
+ return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit, storageBehavior, startTime, timeout);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
new file mode 100644
index 0000000..7fa93e7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+/**
+ */
+public enum StorageSideBehavior {
+ RAW_SCAN, //on use RegionScanner to scan raw data, for testing hbase scan speed
+ SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
+ SCAN_FILTER_AGGR, //aggregate the result. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index fc2fd52..0efea64 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -40,7 +40,7 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
@@ -140,7 +140,7 @@ public class ITKylinQueryTest extends KylinTestBase {
});
Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
BackdoorToggles.setToggles(toggles);
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
deleted file mode 100644
index 5f21351..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.common.coprocessor;
-
-/**
- */
-public enum CoprocessorBehavior {
- RAW_SCAN, //on use RegionScanner to scan raw data, for testing hbase scan speed
- SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
- SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
- SCAN_FILTER_AGGR, //aggregate the result. Will return results
- SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
- SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index c7b650a..7139ca7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -85,15 +85,15 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
- CoprocessorBehavior coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ StorageSideBehavior storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
try {
byte[] behavior = scan.getAttribute(BEHAVIOR);
if (behavior != null && behavior.length != 0) {
- coprocessorBehavior = CoprocessorBehavior.valueOf(new String(behavior));
+ storageSideBehavior = StorageSideBehavior.valueOf(new String(behavior));
}
} catch (Exception e) {
LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
- coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
}
// start/end region operation & sync on scanner is suggested by the
@@ -103,7 +103,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner, coprocessorBehavior);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner, storageSideBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index be26142..a77f988 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -39,9 +39,9 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
- private CoprocessorBehavior behavior;
+ private StorageSideBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, CoprocessorBehavior behavior) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, StorageSideBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
@@ -79,23 +79,23 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (behavior == CoprocessorBehavior.SCAN) {
+ if (behavior == StorageSideBehavior.SCAN) {
//touch every byte of the cell so that the cost of scanning will be trully reflected
int endIndex = cell.getRowOffset() + cell.getRowLength();
for (int i = cell.getRowOffset(); i < endIndex; ++i) {
meaninglessByte += cell.getRowArray()[i];
}
} else {
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER.ordinal()) {
if (filter != null && filter.evaluate(tuple) == false)
continue;
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR.ordinal()) {
AggrKey aggKey = projector.getAggrKey(results);
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, results);
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
aggCache.checkMemoryUsage();
}
}
@@ -103,7 +103,7 @@ public class AggregationScanner implements RegionScanner {
}
}
- if (behavior == CoprocessorBehavior.SCAN) {
+ if (behavior == StorageSideBehavior.SCAN) {
System.out.println("meaningless byte is now " + meaninglessByte);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index f0e9bed..394b3e2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -35,7 +35,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -75,14 +75,14 @@ public class ObserverEnabler {
if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
return new ResultScannerAdapter(aggrScanner);
} else {
// debug/profiling purpose
String toggle = BackdoorToggles.getCoprocessorBehavior();
if (toggle == null) {
- toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ toggle = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5b48351..573951b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -47,7 +46,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
@@ -104,10 +102,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@Override
public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException {
- final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
-
- logger.info("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
-
Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard();
short shardNum = shardNumAndBaseShard.getFirst();
short cuboidBaseShard = shardNumAndBaseShard.getSecond();
@@ -130,39 +124,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
//TODO: raw scan can be constructed at region side to reduce traffic
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
- int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
- while (true) {
- try {
- ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize);
- BytesUtil.writeVInt(rawScans.size(), rawScanBuffer);
- for (RawScan rs : rawScans) {
- RawScan.serializer.serialize(rs, rawScanBuffer);
- }
- rawScanBuffer.flip();
- rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit());
- break;
- } catch (BufferOverflowException boe) {
- logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize);
- rawScanBufferSize *= 4;
- }
- }
+ rawScanByteString = serializeRawScans(rawScans);
+
scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
-
- int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
- while (true) {
- try {
- ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize);
- GTScanRequest.serializer.serialize(scanRequest, buffer);
- buffer.flip();
- scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit());
- break;
- } catch (BufferOverflowException boe) {
- logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize);
- scanRequestBufferSize *= 4;
- }
- }
-
- logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
+ final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
+ scanRequest.setTimeout(epResultItr.getRpcTimeout());
+ scanRequestByteString = serializeGTScanReq(scanRequest);
+
+ logger.info("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size());
for (RawScan rs : rawScans) {
@@ -172,7 +141,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size());
final AtomicLong totalScannedCount = new AtomicLong(0);
- final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
// KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries
// for different cubes until redeployment of coprocessor jar.
@@ -184,9 +152,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
- builder.setBehavior(toggle);
- builder.setStartTime(System.currentTimeMillis());
- builder.setTimeout(epResultItr.getRpcTimeout());
builder.setKylinProperties(kylinConfig.getConfigAsString());
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
@@ -260,6 +225,45 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return new GTBlobScatter(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
}
+ private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
+ ByteString scanRequestByteString;
+ int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
+ while (true) {
+ try {
+ ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize);
+ GTScanRequest.serializer.serialize(scanRequest, buffer);
+ buffer.flip();
+ scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit());
+ break;
+ } catch (BufferOverflowException boe) {
+ logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize);
+ scanRequestBufferSize *= 4;
+ }
+ }
+ return scanRequestByteString;
+ }
+
+ private ByteString serializeRawScans(List<RawScan> rawScans) {
+ ByteString rawScanByteString;
+ int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
+ while (true) {
+ try {
+ ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize);
+ BytesUtil.writeVInt(rawScans.size(), rawScanBuffer);
+ for (RawScan rs : rawScans) {
+ RawScan.serializer.serialize(rs, rawScanBuffer);
+ }
+ rawScanBuffer.flip();
+ rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit());
+ break;
+ } catch (BufferOverflowException boe) {
+ logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize);
+ rawScanBufferSize *= 4;
+ }
+ }
+ return rawScanByteString;
+ }
+
private String getStatsString(byte[] region, CubeVisitResponse result) {
StringBuilder sb = new StringBuilder();
Stats stats = result.getStats();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 442963f..f4729a3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -92,8 +92,8 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
if (coprocException instanceof GTScanSelfTerminatedException)
throw (GTScanSelfTerminatedException) coprocException;
else
- throw new RuntimeException("Error in coprocessor",coprocException);
-
+ throw new RuntimeException("Error in coprocessor", coprocException);
+
} else if (ret == null) {
throw new RuntimeException("Timeout visiting cube! Check why coprocessor exception is not sent back? In coprocessor Self-termination is checked every " + //
GTScanRequest.terminateCheckInterval + " scanned rows, the configured timeout(" + timeout + ") cannot support this many scans?");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 064d100..36adca1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -54,11 +54,11 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanTimeoutException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.BufferedMeasureEncoder;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.CellListIterator;
import org.apache.kylin.storage.hbase.cube.v2.CubeHBaseRPC;
import org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore;
@@ -198,10 +198,10 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
for (IntList intList : request.getHbaseColumnsToGTList()) {
hbaseColumnsToGT.add(intList.getIntsList());
}
- CoprocessorBehavior behavior = CoprocessorBehavior.valueOf(request.getBehavior());
+ StorageSideBehavior behavior = StorageSideBehavior.valueOf(scanReq.getStorageBehavior());
final List<RawScan> hbaseRawScans = deserializeRawScans(ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan())));
- appendProfileInfo(sb, "start latency: " + (this.serviceStartTime - request.getStartTime()));
+ appendProfileInfo(sb, "start latency: " + (this.serviceStartTime - scanReq.getStartTime()));
MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
@Override
@@ -228,7 +228,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
final Iterator<List<Cell>> allCellLists = Iterators.concat(cellListsForeachRawScan.iterator());
- if (behavior.ordinal() < CoprocessorBehavior.SCAN.ordinal()) {
+ if (behavior.ordinal() < StorageSideBehavior.SCAN.ordinal()) {
//this is only for CoprocessorBehavior.RAW_SCAN case to profile hbase scan speed
List<Cell> temp = Lists.newArrayList();
int counter = 0;
@@ -240,12 +240,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
appendProfileInfo(sb, "scanned " + counter);
}
- if (behavior.ordinal() < CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ if (behavior.ordinal() < StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
scanReq.disableAggCacheMemCheck(); // disable mem check if so told
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
- final long deadline = request.getTimeout() + this.serviceStartTime;
+ final long deadline = scanReq.getTimeout() + this.serviceStartTime;
final long storagePushDownLimit = scanReq.getStoragePushDownLimit();
final CellListIterator cellListIterator = new CellListIterator() {
@@ -285,12 +285,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
};
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, //
- request.getRowkeyPreambleSize(), CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(request.getBehavior()));
+ request.getRowkeyPreambleSize(), StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(scanReq.getStorageBehavior()));
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, //
- behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal(), //
- behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal(), deadline);
+ behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER.ordinal(), //
+ behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR.ordinal(), deadline);
ByteBuffer buffer = ByteBuffer.allocate(BufferedMeasureEncoder.DEFAULT_BUFFER_SIZE);
[27/35] kylin git commit: minor, fix CI
Posted by li...@apache.org.
minor, fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e95abde
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e95abde
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e95abde
Branch: refs/heads/1.5.x-HBase1.x
Commit: 5e95abdec6da71a609cea80fa6569328b0c1e6cf
Parents: 3450c0f
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Sep 10 19:26:29 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Sep 10 19:26:42 2016 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e95abde/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 3411c91..de68c7a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -121,6 +121,11 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
+ if (HBaseStorage.overwriteStorageQuery != null) {
+ //v1 engine does not suit
+ return;
+ }
+
thrown.expect(SQLException.class);
//should not break at table duplicate check, should fail at model duplicate check
[11/35] kylin git commit: KYLIN 1997 add pivot feature back in query
result page
Posted by li...@apache.org.
KYLIN 1997 add pivot feature back in query result page
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be7751bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be7751bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be7751bc
Branch: refs/heads/1.5.x-HBase1.x
Commit: be7751bc021063f89e5955e5f1c50d66bedc5b91
Parents: ded3b58
Author: chenzhx <34...@qq.com>
Authored: Thu Sep 8 16:21:32 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu Sep 8 17:51:28 2016 +0800
----------------------------------------------------------------------
webapp/app/js/app.js | 2 +-
webapp/app/partials/query/query_detail.html | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/be7751bc/webapp/app/js/app.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/app.js b/webapp/app/js/app.js
index 3708037..629617e 100644
--- a/webapp/app/js/app.js
+++ b/webapp/app/js/app.js
@@ -17,4 +17,4 @@
*/
//Kylin Application Module
-KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.grid', 'ui.grid.resizeColumns', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable', 'angularBootstrapNavTree', 'toggle-switch', 'ngSanitize', 'ui.select', 'ui.bootstrap.datetimepicker']);
+KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.grid', 'ui.grid.resizeColumns', 'ui.grid.grouping', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable', 'angularBootstrapNavTree', 'toggle-switch', 'ngSanitize', 'ui.select', 'ui.bootstrap.datetimepicker']);
http://git-wip-us.apache.org/repos/asf/kylin/blob/be7751bc/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index 72da6b7..8e1286c 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -96,7 +96,7 @@
<div ng-if="!curQuery.graph.show">
<div class="query-results">
<div ng-if="curQuery.status=='success'">
- <div class="gridStyle" ui-grid="curQuery.result.gridOptions" ui-grid-resize-columns
+ <div class="gridStyle" ui-grid="curQuery.result.gridOptions" ui-grid-resize-columns ui-grid-grouping
ng-if="curQuery.result.results.length > 0" id="data_grid"
style="{{ui.fullScreen?'height: 600px;width:auto':'height: 300px'}}"></div>
<div ng-if="!curQuery.result.results || curQuery.result.results.length == 0"
[28/35] kylin git commit: KYLIN-1983 add license header
Posted by li...@apache.org.
KYLIN-1983 add license header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b941f115
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b941f115
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b941f115
Branch: refs/heads/1.5.x-HBase1.x
Commit: b941f115d5dae06397605c73cb630fbc31c9a5ab
Parents: 5e95abd
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 09:29:45 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
.../MultipleDictionaryValueEnumeratorTest.java | 18 ++++++++++++++++++
.../localmeta/kylin_account.properties | 17 +++++++++++++++++
2 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b941f115/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 6e0f88a..ed8a3c2 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import org.apache.kylin.common.util.Bytes;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b941f115/examples/test_case_data/localmeta/kylin_account.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_account.properties b/examples/test_case_data/localmeta/kylin_account.properties
index 67bbb16..ac34172 100644
--- a/examples/test_case_data/localmeta/kylin_account.properties
+++ b/examples/test_case_data/localmeta/kylin_account.properties
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
### JOB ###
# Only necessary when kylin.job.run.as.remote.cmd=true
[32/35] kylin git commit: KYLIN-1827 code format
Posted by li...@apache.org.
KYLIN-1827 code format
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aef7869c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aef7869c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aef7869c
Branch: refs/heads/1.5.x-HBase1.x
Commit: aef7869c953dec1caaf4b0e426b77c0a9a8938b1
Parents: 0954176
Author: shaofengshi <sh...@apache.org>
Authored: Mon Sep 12 11:36:52 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Sep 12 11:36:52 2016 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 20 ++++++++++----------
.../org/apache/kylin/engine/mr/CubingJob.java | 2 +-
2 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aef7869c/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 1eee5da..90e4d3c 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -125,10 +125,10 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
onExecuteFinished(result, executableContext);
} catch (Exception e) {
- if (isMetaDataPersistException(e)){
+ if (isMetaDataPersistException(e)) {
handleMetaDataPersistException(e);
}
- if (e instanceof ExecuteException){
+ if (e instanceof ExecuteException) {
throw e;
} else {
throw new ExecuteException(e);
@@ -142,15 +142,15 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
}
private boolean isMetaDataPersistException(Exception e) {
- if (e instanceof PersistentException){
+ if (e instanceof PersistentException) {
return true;
}
Throwable t = e.getCause();
int depth = 0;
- while (t!= null && depth<5) {
- depth ++;
- if (t instanceof PersistentException){
+ while (t != null && depth < 5) {
+ depth++;
+ if (t instanceof PersistentException) {
return true;
}
t = t.getCause();
@@ -242,19 +242,19 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected final void notifyUserStatusChange(ExecutableContext context, ExecutableState state) {
try {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- List<String> users = getAllNofifyUsers(kylinConfig);
+ List<String> users = getAllNofifyUsers(kylinConfig);
if (users.isEmpty()) {
logger.warn("no need to send email, user list is empty");
return;
}
final Pair<String, String> email = formatNotifications(context, state);
- doSendMail(kylinConfig,users,email);
+ doSendMail(kylinConfig, users, email);
} catch (Exception e) {
logger.error("error send email", e);
}
}
- private List<String> getAllNofifyUsers(KylinConfig kylinConfig){
+ private List<String> getAllNofifyUsers(KylinConfig kylinConfig) {
List<String> users = Lists.newArrayList();
users.addAll(getNotifyList());
final String[] adminDls = kylinConfig.getAdminDls();
@@ -266,7 +266,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
return users;
}
- private void doSendMail(KylinConfig kylinConfig, List<String> users, Pair<String,String> email){
+ private void doSendMail(KylinConfig kylinConfig, List<String> users, Pair<String, String> email) {
if (email == null) {
logger.warn("no need to send email, content is null");
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/aef7869c/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 9c7f57a..1a0113d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -220,7 +220,7 @@ public class CubingJob extends DefaultChainedExecutable {
} catch (UnknownHostException e) {
logger.warn(e.getLocalizedMessage(), e);
}
- sendMail(Pair.of(title,content));
+ sendMail(Pair.of(title, content));
}
public long getMapReduceWaitTime() {
[13/35] kylin git commit: KYLIN-1996 Keep original column order when
designing cube
Posted by li...@apache.org.
KYLIN-1996 Keep original column order when designing cube
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e87c816d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e87c816d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e87c816d
Branch: refs/heads/1.5.x-HBase1.x
Commit: e87c816dd7099290220ca09361f9ca04c36a317e
Parents: bf26114
Author: chenzhx <34...@qq.com>
Authored: Fri Sep 9 10:55:36 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Sep 9 15:52:48 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeDimensions.js | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e87c816d/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index 1663b5c..ab07451 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -67,20 +67,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var cols = $scope.getDimColumnsByTable(factTable);
// Initialize selected available.
- var factAvailable = {};
var factSelectAvailable = {};
for (var i = 0; i < cols.length; i++) {
cols[i].table = factTable;
cols[i].isLookup = false;
- factAvailable[cols[i].name] = cols[i];
-
// Default not selected and not disabled.
factSelectAvailable[cols[i].name] = {selected: false, disabled: false};
}
- $scope.availableColumns[factTable] = factAvailable;
+ $scope.availableColumns[factTable] = cols;
$scope.selectedColumns[factTable] = factSelectAvailable;
$scope.availableTables.push(factTable);
@@ -91,20 +88,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var cols2 = $scope.getDimColumnsByTable(lookups[j].table);
// Initialize selected available.
- var lookupAvailable = {};
var lookupSelectAvailable = {};
for (var k = 0; k < cols2.length; k++) {
cols2[k].table = lookups[j].table;
cols2[k].isLookup = true;
- lookupAvailable[cols2[k].name] = cols2[k];
-
// Default not selected and not disabled.
lookupSelectAvailable[cols2[k].name] = {selected: false, disabled: false};
}
- $scope.availableColumns[lookups[j].table] = lookupAvailable;
+ $scope.availableColumns[lookups[j].table] = cols2;
$scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
if($scope.availableTables.indexOf(lookups[j].table)==-1){
$scope.availableTables.push(lookups[j].table);
[20/35] kylin git commit: KYLIN-1922 imporve CI
Posted by li...@apache.org.
KYLIN-1922 imporve CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/942406bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/942406bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/942406bd
Branch: refs/heads/1.5.x-HBase1.x
Commit: 942406bda3ec7405a6d2be27ba11bb38b5f88298
Parents: 466cf1a
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 23:01:00 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 23:01:00 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITKylinQueryTest.java | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/942406bd/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5f6af7a..3411c91 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -44,6 +44,8 @@ import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -119,6 +121,22 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
+ thrown.expect(SQLException.class);
+
+ //should not break at table duplicate check, should fail at model duplicate check
+ thrown.expectCause(new BaseMatcher<Throwable>() {
+ @Override
+ public boolean matches(Object item) {
+ if (item instanceof GTScanSelfTerminatedException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ }
+ });
try {
@@ -133,10 +151,6 @@ public class ITKylinQueryTest extends KylinTestBase {
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
- } catch (SQLException e) {
- if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
- throw new RuntimeException();
- }
} finally {
//these two cubes has RAW measure, will disturb limit push down
[25/35] kylin git commit: KYLIN-2004 check whether source data is
empty
Posted by li...@apache.org.
KYLIN-2004 check whether source data is empty
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56136ede
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56136ede
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56136ede
Branch: refs/heads/1.5.x-HBase1.x
Commit: 56136ede7c8b9abac5ddd7b7785b3f63c59b74db
Parents: 233a699
Author: shaofengshi <sh...@apache.org>
Authored: Sat Sep 10 17:52:32 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 17:59:59 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/source/hive/HiveMRInput.java | 37 ++++++++++----------
1 file changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56136ede/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 3ea9af5..520d7cc 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -281,23 +281,6 @@ public class HiveMRInput implements IMRInput {
}
}
- private int determineNumReducer(KylinConfig config) throws IOException {
- computeRowCount(config.getCliCommandExecutor());
-
- Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
- long rowCount = readRowCountFromFile(rowCountFile);
- int mapperInputRows = config.getHadoopJobMapperInputRows();
-
- int numReducers = Math.round(rowCount / ((float) mapperInputRows));
- numReducers = Math.max(1, numReducers);
-
- stepLogger.log("total input rows = " + rowCount);
- stepLogger.log("expected input rows per mapper = " + mapperInputRows);
- stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
-
- return numReducers;
- }
-
private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(getInitStatement());
@@ -327,7 +310,25 @@ public class HiveMRInput implements IMRInput {
KylinConfig config = getCubeSpecificConfig();
try {
- int numReducers = determineNumReducer(config);
+
+ computeRowCount(config.getCliCommandExecutor());
+
+ Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
+ long rowCount = readRowCountFromFile(rowCountFile);
+ if (!config.isEmptySegmentAllowed() && rowCount == 0) {
+ stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+
+ int mapperInputRows = config.getHadoopJobMapperInputRows();
+
+ int numReducers = Math.round(rowCount / ((float) mapperInputRows));
+ numReducers = Math.max(1, numReducers);
+
+ stepLogger.log("total input rows = " + rowCount);
+ stepLogger.log("expected input rows per mapper = " + mapperInputRows);
+ stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
+
redistributeTable(config, numReducers);
return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
[31/35] kylin git commit: KYLIN-1827 Send mail notification when
runtime exception throws during build/merge cube
Posted by li...@apache.org.
KYLIN-1827 Send mail notification when runtime exception throws during build/merge cube
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0954176a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0954176a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0954176a
Branch: refs/heads/1.5.x-HBase1.x
Commit: 0954176adb50df25d0995a7fbeb68e64ee7aed79
Parents: c59d63d
Author: Ma,Gang <ga...@ebay.com>
Authored: Fri Jul 15 11:27:40 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Sep 12 11:23:19 2016 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 123 ++++++++++++++-----
.../org/apache/kylin/engine/mr/CubingJob.java | 40 ++++++
2 files changed, 129 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0954176a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 09f9b54..1eee5da 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -30,6 +30,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.MailService;
import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.exception.PersistentException;
import org.apache.kylin.job.impl.threadpool.DefaultContext;
import org.apache.kylin.job.manager.ExecutableManager;
import org.slf4j.Logger;
@@ -99,31 +100,62 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
Preconditions.checkArgument(executableContext instanceof DefaultContext);
ExecuteResult result = null;
+ try {
+ onExecuteStart(executableContext);
+ Throwable exception;
+ do {
+ if (retry > 0) {
+ logger.info("Retry " + retry);
+ }
+ exception = null;
+ result = null;
+ try {
+ result = doWork(executableContext);
+ } catch (Throwable e) {
+ logger.error("error running Executable: " + this.toString());
+ exception = e;
+ }
+ retry++;
+ } while (((result != null && result.succeed() == false) || exception != null) && needRetry() == true);
- onExecuteStart(executableContext);
- Throwable exception;
- do {
- if (retry > 0) {
- logger.info("Retry " + retry);
+ if (exception != null) {
+ onExecuteError(exception, executableContext);
+ throw new ExecuteException(exception);
}
- exception = null;
- result = null;
- try {
- result = doWork(executableContext);
- } catch (Throwable e) {
- logger.error("error running Executable: " + this.toString());
- exception = e;
+
+ onExecuteFinished(result, executableContext);
+ } catch (Exception e) {
+ if (isMetaDataPersistException(e)){
+ handleMetaDataPersistException(e);
+ }
+ if (e instanceof ExecuteException){
+ throw e;
+ } else {
+ throw new ExecuteException(e);
}
- retry++;
- } while (((result != null && result.succeed() == false) || exception != null) && needRetry() == true);
+ }
+ return result;
+ }
+
+ protected void handleMetaDataPersistException(Exception e) {
+ // do nothing.
+ }
- if (exception != null) {
- onExecuteError(exception, executableContext);
- throw new ExecuteException(exception);
+ private boolean isMetaDataPersistException(Exception e) {
+ if (e instanceof PersistentException){
+ return true;
}
- onExecuteFinished(result, executableContext);
- return result;
+ Throwable t = e.getCause();
+ int depth = 0;
+ while (t!= null && depth<5) {
+ depth ++;
+ if (t instanceof PersistentException){
+ return true;
+ }
+ t = t.getCause();
+ }
+ return false;
}
protected abstract ExecuteResult doWork(ExecutableContext context) throws ExecuteException;
@@ -209,29 +241,52 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected final void notifyUserStatusChange(ExecutableContext context, ExecutableState state) {
try {
- List<String> users = Lists.newArrayList();
- users.addAll(getNotifyList());
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final String[] adminDls = kylinConfig.getAdminDls();
- if (null != adminDls) {
- for (String adminDl : adminDls) {
- users.add(adminDl);
- }
- }
+ List<String> users = getAllNofifyUsers(kylinConfig);
if (users.isEmpty()) {
logger.warn("no need to send email, user list is empty");
return;
}
final Pair<String, String> email = formatNotifications(context, state);
- if (email == null) {
- logger.warn("no need to send email, content is null");
+ doSendMail(kylinConfig,users,email);
+ } catch (Exception e) {
+ logger.error("error send email", e);
+ }
+ }
+
+ private List<String> getAllNofifyUsers(KylinConfig kylinConfig){
+ List<String> users = Lists.newArrayList();
+ users.addAll(getNotifyList());
+ final String[] adminDls = kylinConfig.getAdminDls();
+ if (null != adminDls) {
+ for (String adminDl : adminDls) {
+ users.add(adminDl);
+ }
+ }
+ return users;
+ }
+
+ private void doSendMail(KylinConfig kylinConfig, List<String> users, Pair<String,String> email){
+ if (email == null) {
+ logger.warn("no need to send email, content is null");
+ return;
+ }
+ logger.info("prepare to send email to:" + users);
+ logger.info("job name:" + getName());
+ logger.info("submitter:" + getSubmitter());
+ logger.info("notify list:" + users);
+ new MailService(kylinConfig).sendMail(users, email.getLeft(), email.getRight());
+ }
+
+ protected void sendMail(Pair<String, String> email) {
+ try {
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ List<String> users = getAllNofifyUsers(kylinConfig);
+ if (users.isEmpty()) {
+ logger.warn("no need to send email, user list is empty");
return;
}
- logger.info("prepare to send email to:" + users);
- logger.info("job name:" + getName());
- logger.info("submitter:" + getSubmitter());
- logger.info("notify list:" + users);
- new MailService(kylinConfig).sendMail(users, email.getLeft(), email.getRight());
+ doSendMail(kylinConfig, users, email);
} catch (Exception e) {
logger.error("error send email", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0954176a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index c9ffe14..9c7f57a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -18,6 +18,8 @@
package org.apache.kylin.engine.mr;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
@@ -183,6 +185,44 @@ public class CubingJob extends DefaultChainedExecutable {
super.onExecuteFinished(result, executableContext);
}
+ /**
+ * build fail because the metadata store has problem.
+ * @param exception
+ */
+ @Override
+ protected void handleMetaDataPersistException(Exception exception) {
+ String title = "[ERROR] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + CubingExecutableUtil.getCubeName(this.getParams());
+ String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
+ final String UNKNOWN = "UNKNOWN";
+ String errMsg = null;
+ if (exception != null) {
+ final StringWriter out = new StringWriter();
+ exception.printStackTrace(new PrintWriter(out));
+ errMsg = out.toString();
+ }
+
+ content = content.replaceAll("\\$\\{job_name\\}", getName());
+ content = content.replaceAll("\\$\\{result\\}", ExecutableState.ERROR.toString());
+ content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName());
+ content = content.replaceAll("\\$\\{project_name\\}", getProjectName());
+ content = content.replaceAll("\\$\\{cube_name\\}", CubingExecutableUtil.getCubeName(this.getParams()));
+ content = content.replaceAll("\\$\\{source_records_count\\}", UNKNOWN);
+ content = content.replaceAll("\\$\\{start_time\\}", UNKNOWN);
+ content = content.replaceAll("\\$\\{duration\\}", UNKNOWN);
+ content = content.replaceAll("\\$\\{mr_waiting\\}", UNKNOWN);
+ content = content.replaceAll("\\$\\{last_update_time\\}", UNKNOWN);
+ content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
+ content = content.replaceAll("\\$\\{error_log\\}", Matcher.quoteReplacement(StringUtil.noBlank(errMsg, "no error message")));
+
+ try {
+ InetAddress inetAddress = InetAddress.getLocalHost();
+ content = content.replaceAll("\\$\\{job_engine\\}", inetAddress.getCanonicalHostName());
+ } catch (UnknownHostException e) {
+ logger.warn(e.getLocalizedMessage(), e);
+ }
+ sendMail(Pair.of(title,content));
+ }
+
public long getMapReduceWaitTime() {
return getExtraInfoAsLong(MAP_REDUCE_WAIT_TIME, 0L);
}
[15/35] kylin git commit: KYLIN-1922 optimize needStorageAggregation
check logic and make sure self-termination in coprocessor works
Posted by li...@apache.org.
KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e38557b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e38557b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e38557b4
Branch: refs/heads/1.5.x-HBase1.x
Commit: e38557b4d1cd1d42fe042e5500020cbfaba2d80b
Parents: e87c816
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 15:57:25 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 16:42:33 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 15 +-
.../apache/kylin/cube/RawQueryLastHacker.java | 7 +-
.../cube/gridtable/CubeScanRangePlanner.java | 340 ----------
.../kylin/gridtable/GTAggregateScanner.java | 10 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 6 +-
.../GTScanExceedThresholdException.java | 2 +-
.../apache/kylin/gridtable/GTScanRequest.java | 34 +-
.../GTScanSelfTerminatedException.java | 26 +
.../kylin/gridtable/GTScanTimeoutException.java | 2 +-
.../gridtable/AggregationCacheSpillTest.java | 6 +-
.../kylin/gridtable/DictGridTableTest.java | 617 ------------------
.../storage/gtrecord/CubeScanRangePlanner.java | 357 +++++++++++
.../storage/gtrecord/CubeSegmentScanner.java | 14 +-
.../gtrecord/GTCubeStorageQueryBase.java | 36 +-
.../storage/gtrecord/DictGridTableTest.java | 626 +++++++++++++++++++
.../apache/kylin/query/ITKylinQueryTest.java | 55 +-
.../resources/query/sql_timeout/query01.sql | 19 +
.../common/coprocessor/CoprocessorBehavior.java | 1 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 8 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/ExpectedSizeIterator.java | 39 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 11 +-
.../coprocessor/endpoint/CubeVisitService.java | 26 +-
23 files changed, 1197 insertions(+), 1062 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f0c91da..2ac9d48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,8 +481,8 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
- public int getCubeVisitTimeoutTimes() {
- return Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
+ public float getCubeVisitTimeoutTimes() {
+ return Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
}
public int getBadQueryStackTraceDepth() {
@@ -545,15 +545,6 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false"));
}
- public String getQueryStorageVisitPlanner() {
- return this.getOptional("kylin.query.storage.visit.planner", "org.apache.kylin.cube.gridtable.CubeScanRangePlanner");
- }
-
- // for test only
- public void setQueryStorageVisitPlanner(String v) {
- setProperty("kylin.query.storage.visit.planner", v);
- }
-
public int getQueryScanFuzzyKeyMax() {
return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
}
@@ -573,7 +564,7 @@ abstract public class KylinConfigBase implements Serializable {
public boolean getQueryMetricsEnabled() {
return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false"));
}
-
+
public int[] getQueryMetricsPercentilesIntervals() {
String[] dft = { "60", "300", "3600" };
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 63ddac5..50c644e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -44,13 +44,14 @@ public class RawQueryLastHacker {
// We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
for (TblColRef col : cubeDesc.listAllColumns()) {
- if (col.getTable().equals(sqlDigest.factTable) && (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll)) {
- sqlDigest.allColumns.add(col);
+ if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll) {
+ if (col.getTable().equals(sqlDigest.factTable))
+ sqlDigest.allColumns.add(col);
}
}
for (TblColRef col : sqlDigest.allColumns) {
- if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
+ if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) {
// For dimension columns, take them as group by columns.
sqlDigest.groupbyColumns.add(col);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
deleted file mode 100644
index a937045..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.common.FuzzyValueCombination;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.gridtable.GTScanRange;
-import org.apache.kylin.gridtable.GTScanRequest;
-import org.apache.kylin.gridtable.GTScanRequestBuilder;
-import org.apache.kylin.gridtable.GTUtil;
-import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-public class CubeScanRangePlanner extends ScanRangePlannerBase {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
-
- protected int maxScanRanges;
- protected int maxFuzzyKeys;
-
- //non-GT
- protected CubeSegment cubeSegment;
- protected CubeDesc cubeDesc;
- protected Cuboid cuboid;
-
- public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
- Collection<FunctionDesc> metrics) {
-
- this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
- this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
-
- this.cubeSegment = cubeSegment;
- this.cubeDesc = cubeSegment.getCubeDesc();
- this.cuboid = cuboid;
-
- Set<TblColRef> filterDims = Sets.newHashSet();
- TupleFilter.collectColumns(filter, filterDims);
-
- this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
- CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
-
- IGTComparator comp = gtInfo.getCodeSystem().getComparator();
- //start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
- //stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
- //start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
-
- //replace the constant values in filter to dictionary codes
- this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
-
- this.gtDimensions = mapping.makeGridTableColumns(dimensions);
- this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
- this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
- this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
-
- if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
- int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
- if (index >= 0) {
- SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
- this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
- this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
- this.gtPartitionCol = gtInfo.colRef(index);
- }
- }
-
- }
-
- /**
- * constrcut GTScanRangePlanner with incomplete information. only be used for UT
- * @param info
- * @param gtStartAndEnd
- * @param gtPartitionCol
- * @param gtFilter
- */
- public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
-
- this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
- this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
-
- this.gtInfo = info;
-
- IGTComparator comp = gtInfo.getCodeSystem().getComparator();
- //start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
- //stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
- //start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
-
- this.gtFilter = gtFilter;
- this.gtStartAndEnd = gtStartAndEnd;
- this.gtPartitionCol = gtPartitionCol;
- }
-
- public GTScanRequest planScanRequest() {
- GTScanRequest scanRequest;
- List<GTScanRange> scanRanges = this.planScanRanges();
- if (scanRanges != null && scanRanges.size() != 0) {
- scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).createGTScanRequest();
- } else {
- scanRequest = null;
- }
- return scanRequest;
- }
-
- /**
- * Overwrite this method to provide smarter storage visit plans
- * @return
- */
- public List<GTScanRange> planScanRanges() {
- TupleFilter flatFilter = flattenToOrAndFilter(gtFilter);
-
- List<Collection<ColumnRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter);
-
- List<GTScanRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
- for (Collection<ColumnRange> andDimRanges : orAndDimRanges) {
- GTScanRange scanRange = newScanRange(andDimRanges);
- if (scanRange != null)
- scanRanges.add(scanRange);
- }
-
- List<GTScanRange> mergedRanges = mergeOverlapRanges(scanRanges);
- mergedRanges = mergeTooManyRanges(mergedRanges, maxScanRanges);
-
- return mergedRanges;
- }
-
- private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
- Set<TblColRef> ret = Sets.newHashSet();
- for (TblColRef col : input) {
- if (cubeDesc.hasHostColumn(col)) {
- for (TblColRef host : cubeDesc.getHostInfo(col).columns) {
- ret.add(host);
- }
- } else {
- ret.add(col);
- }
- }
- return ret;
- }
-
- protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
- GTRecord pkStart = new GTRecord(gtInfo);
- GTRecord pkEnd = new GTRecord(gtInfo);
- Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
-
- List<GTRecord> fuzzyKeys;
-
- for (ColumnRange range : andDimRanges) {
- if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
- int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
- int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
-
- if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
- } else {
- logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
- gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
- return null;
- }
- }
-
- int col = range.column.getColumnDesc().getZeroBasedIndex();
- if (!gtInfo.getPrimaryKey().get(col))
- continue;
-
- pkStart.set(col, range.begin);
- pkEnd.set(col, range.end);
-
- if (range.valueSet != null && !range.valueSet.isEmpty()) {
- fuzzyValues.put(col, range.valueSet);
- }
- }
-
- fuzzyKeys =
-
- buildFuzzyKeys(fuzzyValues);
- return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
- }
-
- private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
- ArrayList<GTRecord> result = Lists.newArrayList();
-
- if (fuzzyValueSet.isEmpty())
- return result;
-
- // debug/profiling purpose
- if (BackdoorToggles.getDisableFuzzyKey()) {
- logger.info("The execution of this query will not use fuzzy key");
- return result;
- }
-
- List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKeys);
-
- for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
-
- // BitSet bitSet = new BitSet(gtInfo.getColumnCount());
- // for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
- // bitSet.set(entry.getKey());
- // }
- GTRecord fuzzy = new GTRecord(gtInfo);
- for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
- fuzzy.set(entry.getKey(), entry.getValue());
- }
-
- result.add(fuzzy);
- }
- return result;
- }
-
- protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
- if (ranges.size() <= 1) {
- return ranges;
- }
-
- // sort ranges by start key
- Collections.sort(ranges, new Comparator<GTScanRange>() {
- @Override
- public int compare(GTScanRange a, GTScanRange b) {
- return rangeStartComparator.compare(a.pkStart, b.pkStart);
- }
- });
-
- // merge the overlap range
- List<GTScanRange> mergedRanges = new ArrayList<GTScanRange>();
- int mergeBeginIndex = 0;
- GTRecord mergeEnd = ranges.get(0).pkEnd;
- for (int index = 1; index < ranges.size(); index++) {
- GTScanRange range = ranges.get(index);
-
- // if overlap, swallow it
- if (rangeStartEndComparator.compare(range.pkStart, mergeEnd) <= 0) {
- mergeEnd = rangeEndComparator.max(mergeEnd, range.pkEnd);
- continue;
- }
-
- // not overlap, split here
- GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, index));
- mergedRanges.add(mergedRange);
-
- // start new split
- mergeBeginIndex = index;
- mergeEnd = range.pkEnd;
- }
-
- // don't miss the last range
- GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, ranges.size()));
- mergedRanges.add(mergedRange);
-
- return mergedRanges;
- }
-
- private GTScanRange mergeKeyRange(List<GTScanRange> ranges) {
- GTScanRange first = ranges.get(0);
- if (ranges.size() == 1)
- return first;
-
- GTRecord start = first.pkStart;
- GTRecord end = first.pkEnd;
- List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
-
- boolean hasNonFuzzyRange = false;
- for (GTScanRange range : ranges) {
- hasNonFuzzyRange = hasNonFuzzyRange || range.fuzzyKeys.isEmpty();
- newFuzzyKeys.addAll(range.fuzzyKeys);
- end = rangeEndComparator.max(end, range.pkEnd);
- }
-
- // if any range is non-fuzzy, then all fuzzy keys must be cleared
- // also too many fuzzy keys will slow down HBase scan
- if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKeys) {
- newFuzzyKeys.clear();
- }
-
- return new GTScanRange(start, end, newFuzzyKeys);
- }
-
- protected List<GTScanRange> mergeTooManyRanges(List<GTScanRange> ranges, int maxRanges) {
- if (ranges.size() <= maxRanges) {
- return ranges;
- }
-
- // TODO: check the distance between range and merge the large distance range
- List<GTScanRange> result = new ArrayList<GTScanRange>(1);
- GTScanRange mergedRange = mergeKeyRange(ranges);
- result.add(mergedRange);
- return result;
- }
-
- public int getMaxScanRanges() {
- return maxScanRanges;
- }
-
- public void setMaxScanRanges(int maxScanRanges) {
- this.maxScanRanges = maxScanRanges;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index ccf4895..db38484 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -138,7 +138,10 @@ public class GTAggregateScanner implements IGTScanner {
long count = 0;
for (GTRecord r : inputScanner) {
- count++;
+ //check deadline
+ if (count % GTScanRequest.terminateCheckInterval == 1 && System.currentTimeMillis() > deadline) {
+ throw new GTScanTimeoutException("Timeout in GTAggregateScanner with scanned count " + count);
+ }
if (getNumOfSpills() == 0) {
//check limit
@@ -152,10 +155,7 @@ public class GTAggregateScanner implements IGTScanner {
aggrCache.aggregate(r, Integer.MAX_VALUE);
}
- //check deadline
- if (count % 10000 == 1 && System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("Timeout in GTAggregateScanner with scanned count " + count);
- }
+ count++;
}
logger.info("GTAggregateScanner input rows: " + count);
return aggrCache.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index 31a9599..f1f84af 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -132,12 +132,12 @@ public class GTFilterScanner implements IGTScanner {
}
// cache the last one input and result, can reuse because rowkey are ordered, and same input could come in small group
- static class FilterResultCache {
+ public static class FilterResultCache {
static final int CHECKPOINT = 10000;
static final double HIT_RATE_THRESHOLD = 0.5;
- static boolean ENABLED = true; // enable cache by default
+ public static boolean ENABLED = true; // enable cache by default
- boolean enabled = ENABLED;
+ public boolean enabled = ENABLED;
ImmutableBitSet colsInFilter;
int count;
int hit;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
index dd57e90..ba75962 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
@@ -18,7 +18,7 @@
package org.apache.kylin.gridtable;
-public class GTScanExceedThresholdException extends RuntimeException {
+public class GTScanExceedThresholdException extends GTScanSelfTerminatedException {
public GTScanExceedThresholdException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 4cfba1b..5d27028 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -42,6 +42,8 @@ import com.google.common.collect.Sets;
public class GTScanRequest {
private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
+ //it's not necessary to increase the checkInterval to very large because the check cost is not high
+ public static final int terminateCheckInterval = 1000;
private GTInfo info;
private List<GTScanRange> ranges;
@@ -55,13 +57,16 @@ public class GTScanRequest {
private ImmutableBitSet aggrGroupBy;
private ImmutableBitSet aggrMetrics;
private String[] aggrMetricsFuncs;//
-
+
// hint to storage behavior
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
private int storageScanRowNumThreshold;
private int storagePushDownLimit;
+ // runtime computed fields
+ private transient boolean doingStorageAggregation = false;
+
GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, boolean allowStorageAggregation, //
double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit) {
@@ -169,6 +174,7 @@ public class GTScanRequest {
logger.info("pre aggregation is not beneficial, skip it");
} else if (this.hasAggregation()) {
logger.info("pre aggregating results before returning");
+ this.doingStorageAggregation = true;
result = new GTAggregateScanner(result, this, deadline);
} else {
logger.info("has no aggregation, skip it");
@@ -178,6 +184,10 @@ public class GTScanRequest {
}
+ public boolean isDoingStorageAggregation() {
+ return doingStorageAggregation;
+ }
+
//touch every byte of the cell so that the cost of scanning will be truly reflected
private int lookAndForget(IGTScanner scanner) {
byte meaninglessByte = 0;
@@ -215,8 +225,8 @@ public class GTScanRequest {
return ranges;
}
- public void setGTScanRanges(List<GTScanRange> ranges) {
- this.ranges = ranges;
+ public void clearScanRanges() {
+ this.ranges = Lists.newArrayList();
}
public ImmutableBitSet getSelectedColBlocks() {
@@ -251,10 +261,6 @@ public class GTScanRequest {
return allowStorageAggregation;
}
- public void setAllowStorageAggregation(boolean allowStorageAggregation) {
- this.allowStorageAggregation = allowStorageAggregation;
- }
-
public double getAggCacheMemThreshold() {
if (aggCacheMemThreshold < 0)
return 0;
@@ -262,28 +268,18 @@ public class GTScanRequest {
return aggCacheMemThreshold;
}
- public void setAggCacheMemThreshold(double gb) {
- this.aggCacheMemThreshold = gb;
+ public void disableAggCacheMemCheck() {
+ this.aggCacheMemThreshold = 0;
}
public int getStorageScanRowNumThreshold() {
return storageScanRowNumThreshold;
}
- public void setStorageScanRowNumThreshold(int storageScanRowNumThreshold) {
- logger.info("storageScanRowNumThreshold is set to " + storageScanRowNumThreshold);
- this.storageScanRowNumThreshold = storageScanRowNumThreshold;
- }
-
public int getStoragePushDownLimit() {
return this.storagePushDownLimit;
}
- public void setStoragePushDownLimit(int limit) {
- logger.info("storagePushDownLimit is set to " + storagePushDownLimit);
- this.storagePushDownLimit = limit;
- }
-
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
new file mode 100644
index 0000000..4775ac6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+public class GTScanSelfTerminatedException extends RuntimeException {
+
+ public GTScanSelfTerminatedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
index e92dae3..17a8d02 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
@@ -18,7 +18,7 @@
package org.apache.kylin.gridtable;
-public class GTScanTimeoutException extends RuntimeException {
+public class GTScanTimeoutException extends GTScanSelfTerminatedException {
public GTScanTimeoutException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index b5f6de7..38b8c90 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -84,8 +84,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
};
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).createGTScanRequest();
- scanRequest.setAggCacheMemThreshold(0.5);
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
@@ -127,8 +126,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
};
// all-in-mem testcase
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).createGTScanRequest();
- scanRequest.setAggCacheMemThreshold(0.5);
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
deleted file mode 100644
index 7b6d3fa..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.gridtable.CubeCodeSystem;
-import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
-import org.apache.kylin.dict.NumberDictionaryBuilder;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.apache.kylin.dict.TrieDictionaryBuilder;
-import org.apache.kylin.dimension.DictionaryDimEnc;
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
-import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ExtractTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class DictGridTableTest extends LocalFileMetadataTestCase {
-
- private GridTable table;
- private GTInfo info;
- private CompareTupleFilter timeComp0;
- private CompareTupleFilter timeComp1;
- private CompareTupleFilter timeComp2;
- private CompareTupleFilter timeComp3;
- private CompareTupleFilter timeComp4;
- private CompareTupleFilter timeComp5;
- private CompareTupleFilter timeComp6;
- private CompareTupleFilter ageComp1;
- private CompareTupleFilter ageComp2;
- private CompareTupleFilter ageComp3;
- private CompareTupleFilter ageComp4;
-
- @After
- public void after() throws Exception {
-
- this.cleanupTestMetadata();
- }
-
- @Before
- public void setup() throws IOException {
-
- this.createTestMetadata();
-
- table = newTestTable();
- info = table.getInfo();
-
- timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14"));
- timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
- timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
- timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
- timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15"));
- timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14"));
- ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
- ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
- ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
- ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
-
- }
-
- @Test
- public void verifySegmentSkipping() {
-
- ByteArray segmentStart = enc(info, 0, "2015-01-14");
- ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free
- ByteArray segmentEnd = enc(info, 0, "2015-01-15");
- assertEquals(segmentStart, segmentStartX);
-
- {
- LogicalTupleFilter filter = and(timeComp0, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());//scan range are [close,close]
- assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
- assertEquals(1, r.get(0).fuzzyKeys.size());
- assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
- {
- LogicalTupleFilter filter = and(timeComp2, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = and(timeComp4, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = and(timeComp5, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
- assertEquals("[[null, 10, null, null, null], [1421193600000, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
- {
- LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
- assertEquals(0, r.get(0).fuzzyKeys.size());
- }
- {
- //skip FALSE filter
- LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- //TRUE or FALSE filter
- LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[null, null]-[null, null]", r.get(0).toString());
- }
- {
- //TRUE or other filter
- LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[null, null]-[null, null]", r.get(0).toString());
- }
- }
-
- @Test
- public void verifySegmentSkipping2() {
- ByteArray segmentEnd = enc(info, 0, "2015-01-15");
-
- {
- LogicalTupleFilter filter = and(timeComp0, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());//scan range are [close,close]
- assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
- assertEquals(1, r.get(0).fuzzyKeys.size());
- assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
-
- {
- LogicalTupleFilter filter = and(timeComp5, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());//scan range are [close,close]
- }
- }
-
- @Test
- public void verifyScanRangePlanner() {
-
- // flatten or-and & hbase fuzzy value
- {
- LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
- assertEquals("[[null, 10, null, null, null], [null, 20, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
-
- // pre-evaluate ever false
- {
- LogicalTupleFilter filter = and(timeComp1, timeComp2);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
-
- // pre-evaluate ever true
- {
- LogicalTupleFilter filter = or(timeComp1, ageComp4);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals("[[null, null]-[null, null]]", r.toString());
- }
-
- // merge overlap range
- {
- LogicalTupleFilter filter = or(timeComp1, timeComp3);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals("[[null, null]-[null, null]]", r.toString());
- }
-
- // merge too many ranges
- {
- LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(3, r.size());
- assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
- assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString());
- assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString());
- planner.setMaxScanRanges(2);
- List<GTScanRange> r2 = planner.planScanRanges();
- assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString());
- }
- }
-
- @Test
- public void verifyFirstRow() throws IOException {
- doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", //
- "[1421193600000, 30, Luke, 10, 10.5]", //
- "[1421280000000, 20, Dong, 10, 10.5]", //
- "[1421280000000, 20, Jason, 10, 10.5]", //
- "[1421280000000, 30, Xu, 10, 10.5]", //
- "[1421366400000, 20, Mahone, 10, 10.5]", //
- "[1421366400000, 20, Qianhao, 10, 10.5]", //
- "[1421366400000, 30, George, 10, 10.5]", //
- "[1421366400000, 30, Shaofeng, 10, 10.5]", //
- "[1421452800000, 10, Kejia, 10, 10.5]");
- }
-
- //for testing GTScanRequest serialization and deserialization
- public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
- ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
- GTScanRequest.serializer.serialize(origin, buffer);
- buffer.flip();
- GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer);
-
- Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs());
- Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01);
- return sGTScanRequest;
- }
-
- @Test
- public void verifyScanWithUnevaluatableFilter() throws IOException {
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
- LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
- LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);
-
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[]{"sum"}).setFilterPushDown(filter).createGTScanRequest();
-
- // note the unEvaluatable column 1 in filter is added to group by
- assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
-
- doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
- }
-
- @Test
- public void verifyScanWithEvaluatableFilter() throws IOException {
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[]{"sum"}).setFilterPushDown(filter).createGTScanRequest();
- // note the evaluatable column 1 in filter is added to returned columns but not in group by
- assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
-
- doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]");
- }
-
- @Test
- public void testFilterScannerPerf() throws IOException {
- GridTable table = newTestPerfTable();
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- FilterResultCache.ENABLED = false;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = true;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = false;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = true;
- testFilterScannerPerfInner(table, info, filter);
- }
-
- @SuppressWarnings("unused")
- private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException {
- long start = System.currentTimeMillis();
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(filter).createGTScanRequest();
- IGTScanner scanner = table.scan(req);
- int i = 0;
- for (GTRecord r : scanner) {
- i++;
- }
- scanner.close();
- long end = System.currentTimeMillis();
- System.out.println((end - start) + "ms with filter cache enabled=" + FilterResultCache.ENABLED + ", " + i + " rows");
- }
-
- @Test
- public void verifyConvertFilterConstants1() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants2() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1<"9" round up to $1<"10"
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants3() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1<="9" round down to FALSE
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants4() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1 in ("9", "10", "15") has only "10" left
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString());
- }
-
- private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
- System.out.println(req);
- IGTScanner scanner = table.scan(req);
- int i = 0;
- for (GTRecord r : scanner) {
- System.out.println(r);
- if (verifyRows == null || i >= verifyRows.length) {
- Assert.fail();
- }
- assertEquals(verifyRows[i], r.toString());
- i++;
- }
- scanner.close();
- }
-
- public static ByteArray enc(GTInfo info, int col, String value) {
- ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
- info.codeSystem.encodeColumnValue(col, value, buf);
- return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
- }
-
- public static ExtractTupleFilter unevaluatable(TblColRef col) {
- ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
- r.addChild(new ColumnTupleFilter(col));
- return r;
- }
-
- public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
- CompareTupleFilter result = new CompareTupleFilter(op);
- result.addChild(new ColumnTupleFilter(col));
- result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
- return result;
- }
-
- public static LogicalTupleFilter and(TupleFilter... children) {
- return logic(FilterOperatorEnum.AND, children);
- }
-
- public static LogicalTupleFilter or(TupleFilter... children) {
- return logic(FilterOperatorEnum.OR, children);
- }
-
- public static LogicalTupleFilter not(TupleFilter child) {
- return logic(FilterOperatorEnum.NOT, child);
- }
-
- public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
- LogicalTupleFilter result = new LogicalTupleFilter(op);
- for (TupleFilter c : children) {
- result.addChild(c);
- }
- return result;
- }
-
- public static GridTable newTestTable() throws IOException {
- GTInfo info = newInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder = table.rebuild();
-
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
- builder.close();
-
- return table;
- }
-
- static GridTable newTestPerfTable() throws IOException {
- GTInfo info = newInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder = table.rebuild();
-
- for (int i = 0; i < 100000; i++) {
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
- }
- builder.close();
-
- return table;
- }
-
- static GTInfo newInfo() {
- Builder builder = GTInfo.builder();
- builder.setCodeSystem(newDictCodeSystem());
- builder.setColumns( //
- DataType.getType("timestamp"), //
- DataType.getType("integer"), //
- DataType.getType("varchar(10)"), //
- DataType.getType("bigint"), //
- DataType.getType("decimal") //
- );
- builder.setPrimaryKey(setOf(0, 1));
- builder.setColumnPreferIndex(setOf(0));
- builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
- builder.enableRowBlock(4);
- GTInfo info = builder.build();
- return info;
- }
-
- @SuppressWarnings("unchecked")
- private static CubeCodeSystem newDictCodeSystem() {
- DimensionEncoding[] dimEncs = new DimensionEncoding[3];
- dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
- dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString());
- return new CubeCodeSystem(dimEncs);
- }
-
- @SuppressWarnings("rawtypes")
- private static Dictionary newDictionaryOfString() {
- TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
- builder.addValue("Dong");
- builder.addValue("George");
- builder.addValue("Jason");
- builder.addValue("Kejia");
- builder.addValue("Luke");
- builder.addValue("Mahone");
- builder.addValue("Qianhao");
- builder.addValue("Shaofeng");
- builder.addValue("Xu");
- builder.addValue("Yang");
- return builder.build(0);
- }
-
- @SuppressWarnings("rawtypes")
- private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
- builder.addValue("10");
- builder.addValue("20");
- builder.addValue("30");
- builder.addValue("40");
- builder.addValue("50");
- builder.addValue("60");
- builder.addValue("70");
- builder.addValue("80");
- builder.addValue("90");
- builder.addValue("100");
- return builder.build(0);
- }
-
- public static ImmutableBitSet setOf(int... values) {
- BitSet set = new BitSet();
- for (int i : values)
- set.set(i);
- return new ImmutableBitSet(set);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
new file mode 100644
index 0000000..9f505f3
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.FuzzyValueCombination;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.gridtable.CubeGridTable;
+import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.gridtable.RecordComparators;
+import org.apache.kylin.cube.gridtable.ScanRangePlannerBase;
+import org.apache.kylin.cube.gridtable.SegmentGTStartAndEnd;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.IGTComparator;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CubeScanRangePlanner extends ScanRangePlannerBase {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
+
+ protected int maxScanRanges;
+ protected int maxFuzzyKeys;
+
+ //non-GT
+ protected CubeSegment cubeSegment;
+ protected CubeDesc cubeDesc;
+ protected Cuboid cuboid;
+
+ protected StorageContext context;
+
+ public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
+ Collection<FunctionDesc> metrics, StorageContext context) {
+ this.context = context;
+
+ this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+ this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
+ this.cubeSegment = cubeSegment;
+ this.cubeDesc = cubeSegment.getCubeDesc();
+ this.cuboid = cuboid;
+
+ Set<TblColRef> filterDims = Sets.newHashSet();
+ TupleFilter.collectColumns(filter, filterDims);
+
+ this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
+ CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
+
+ IGTComparator comp = gtInfo.getCodeSystem().getComparator();
+ //start key GTRecord compare to start key GTRecord
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
+ //stop key GTRecord compare to stop key GTRecord
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
+ //start key GTRecord compare to stop key GTRecord
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
+
+ //replace the constant values in filter to dictionary codes
+ this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
+
+ this.gtDimensions = mapping.makeGridTableColumns(dimensions);
+ this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
+ this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
+ this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
+
+ if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
+ int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
+ if (index >= 0) {
+ SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
+ this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
+ this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
+ this.gtPartitionCol = gtInfo.colRef(index);
+ }
+ }
+
+ }
+
+ /**
+ * constrcut GTScanRangePlanner with incomplete information. only be used for UT
+ * @param info
+ * @param gtStartAndEnd
+ * @param gtPartitionCol
+ * @param gtFilter
+ */
+ public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
+
+ this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+ this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
+ this.gtInfo = info;
+
+ IGTComparator comp = gtInfo.getCodeSystem().getComparator();
+ //start key GTRecord compare to start key GTRecord
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
+ //stop key GTRecord compare to stop key GTRecord
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
+ //start key GTRecord compare to stop key GTRecord
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
+
+ this.gtFilter = gtFilter;
+ this.gtStartAndEnd = gtStartAndEnd;
+ this.gtPartitionCol = gtPartitionCol;
+ }
+
+ public GTScanRequest planScanRequest() {
+ GTScanRequest scanRequest;
+ List<GTScanRange> scanRanges = this.planScanRanges();
+ if (scanRanges != null && scanRanges.size() != 0) {
+ GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
+ setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).//
+ setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).//
+ setStorageScanRowNumThreshold(context.getThreshold());
+
+ if (cubeDesc.supportsLimitPushDown()) {
+ builder.setStoragePushDownLimit(context.getStoragePushDownLimit());
+ }
+ scanRequest = builder.createGTScanRequest();
+ } else {
+ scanRequest = null;
+ }
+ return scanRequest;
+ }
+
+ /**
+ * Overwrite this method to provide smarter storage visit plans
+ * @return
+ */
+ public List<GTScanRange> planScanRanges() {
+ TupleFilter flatFilter = flattenToOrAndFilter(gtFilter);
+
+ List<Collection<ColumnRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter);
+
+ List<GTScanRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
+ for (Collection<ColumnRange> andDimRanges : orAndDimRanges) {
+ GTScanRange scanRange = newScanRange(andDimRanges);
+ if (scanRange != null)
+ scanRanges.add(scanRange);
+ }
+
+ List<GTScanRange> mergedRanges = mergeOverlapRanges(scanRanges);
+ mergedRanges = mergeTooManyRanges(mergedRanges, maxScanRanges);
+
+ return mergedRanges;
+ }
+
+ private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
+ Set<TblColRef> ret = Sets.newHashSet();
+ for (TblColRef col : input) {
+ if (cubeDesc.hasHostColumn(col)) {
+ for (TblColRef host : cubeDesc.getHostInfo(col).columns) {
+ ret.add(host);
+ }
+ } else {
+ ret.add(col);
+ }
+ }
+ return ret;
+ }
+
+ protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
+ GTRecord pkStart = new GTRecord(gtInfo);
+ GTRecord pkEnd = new GTRecord(gtInfo);
+ Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
+
+ List<GTRecord> fuzzyKeys;
+
+ for (ColumnRange range : andDimRanges) {
+ if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
+ int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
+ int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
+
+ if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
+ } else {
+ logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
+ gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
+ return null;
+ }
+ }
+
+ int col = range.column.getColumnDesc().getZeroBasedIndex();
+ if (!gtInfo.getPrimaryKey().get(col))
+ continue;
+
+ pkStart.set(col, range.begin);
+ pkEnd.set(col, range.end);
+
+ if (range.valueSet != null && !range.valueSet.isEmpty()) {
+ fuzzyValues.put(col, range.valueSet);
+ }
+ }
+
+ fuzzyKeys =
+
+ buildFuzzyKeys(fuzzyValues);
+ return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
+ }
+
+ private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
+ ArrayList<GTRecord> result = Lists.newArrayList();
+
+ if (fuzzyValueSet.isEmpty())
+ return result;
+
+ // debug/profiling purpose
+ if (BackdoorToggles.getDisableFuzzyKey()) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+
+ List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKeys);
+
+ for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
+
+ // BitSet bitSet = new BitSet(gtInfo.getColumnCount());
+ // for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
+ // bitSet.set(entry.getKey());
+ // }
+ GTRecord fuzzy = new GTRecord(gtInfo);
+ for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
+ fuzzy.set(entry.getKey(), entry.getValue());
+ }
+
+ result.add(fuzzy);
+ }
+ return result;
+ }
+
+ protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
+ if (ranges.size() <= 1) {
+ return ranges;
+ }
+
+ // sort ranges by start key
+ Collections.sort(ranges, new Comparator<GTScanRange>() {
+ @Override
+ public int compare(GTScanRange a, GTScanRange b) {
+ return rangeStartComparator.compare(a.pkStart, b.pkStart);
+ }
+ });
+
+ // merge the overlap range
+ List<GTScanRange> mergedRanges = new ArrayList<GTScanRange>();
+ int mergeBeginIndex = 0;
+ GTRecord mergeEnd = ranges.get(0).pkEnd;
+ for (int index = 1; index < ranges.size(); index++) {
+ GTScanRange range = ranges.get(index);
+
+ // if overlap, swallow it
+ if (rangeStartEndComparator.compare(range.pkStart, mergeEnd) <= 0) {
+ mergeEnd = rangeEndComparator.max(mergeEnd, range.pkEnd);
+ continue;
+ }
+
+ // not overlap, split here
+ GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, index));
+ mergedRanges.add(mergedRange);
+
+ // start new split
+ mergeBeginIndex = index;
+ mergeEnd = range.pkEnd;
+ }
+
+ // don't miss the last range
+ GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, ranges.size()));
+ mergedRanges.add(mergedRange);
+
+ return mergedRanges;
+ }
+
+ private GTScanRange mergeKeyRange(List<GTScanRange> ranges) {
+ GTScanRange first = ranges.get(0);
+ if (ranges.size() == 1)
+ return first;
+
+ GTRecord start = first.pkStart;
+ GTRecord end = first.pkEnd;
+ List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
+
+ boolean hasNonFuzzyRange = false;
+ for (GTScanRange range : ranges) {
+ hasNonFuzzyRange = hasNonFuzzyRange || range.fuzzyKeys.isEmpty();
+ newFuzzyKeys.addAll(range.fuzzyKeys);
+ end = rangeEndComparator.max(end, range.pkEnd);
+ }
+
+ // if any range is non-fuzzy, then all fuzzy keys must be cleared
+ // also too many fuzzy keys will slow down HBase scan
+ if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKeys) {
+ newFuzzyKeys.clear();
+ }
+
+ return new GTScanRange(start, end, newFuzzyKeys);
+ }
+
+ protected List<GTScanRange> mergeTooManyRanges(List<GTScanRange> ranges, int maxRanges) {
+ if (ranges.size() <= maxRanges) {
+ return ranges;
+ }
+
+ // TODO: check the distance between range and merge the large distance range
+ List<GTScanRange> result = new ArrayList<GTScanRange>(1);
+ GTScanRange mergedRange = mergeKeyRange(ranges);
+ result.add(mergedRange);
+ return result;
+ }
+
+ public int getMaxScanRanges() {
+ return maxScanRanges;
+ }
+
+ public void setMaxScanRanges(int maxScanRanges) {
+ this.maxScanRanges = maxScanRanges;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 3b9d9c6..f32831a 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -23,10 +23,8 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -67,23 +65,13 @@ public class CubeSegmentScanner implements IGTScanner {
ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
- String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner();
CubeScanRangePlanner scanRangePlanner;
try {
- scanRangePlanner = (CubeScanRangePlanner) Class.forName(plannerName).getConstructor(CubeSegment.class, Cuboid.class, TupleFilter.class, Set.class, Set.class, Collection.class).newInstance(cubeSeg, cuboid, filter, dimensions, groups, metrics);
+ scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics, context);
} catch (Exception e) {
throw new RuntimeException(e);
}
scanRequest = scanRangePlanner.planScanRequest();
- if (scanRequest != null) {
- scanRequest.setAllowStorageAggregation(context.isNeedStorageAggregation());
- scanRequest.setAggCacheMemThreshold(cubeSeg.getCubeInstance().getConfig().getQueryCoprocessorMemGB());
- scanRequest.setStorageScanRowNumThreshold(context.getThreshold());//TODO: devide by shard number?
-
- if (cubeSeg.getCubeDesc().supportsLimitPushDown()) {
- scanRequest.setStoragePushDownLimit(context.getStoragePushDownLimit());
- }
- }
scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 86346f8..f0c2494 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.gtrecord;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -72,10 +73,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-
+
//cope with queries with no aggregations
RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
-
+
// Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
@@ -112,9 +113,9 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
TupleFilter filterD = translateDerived(filter, groupsD);
- context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD, exactAggregation));
- enableStoragePushDownLimit(cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context);
- setThreshold(dimensionsD, metrics, context); // set cautious threshold to prevent out of memory
+ context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD));
+ enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context);
+ setThresholdIfNecessary(dimensionsD, metrics, context); // set cautious threshold to prevent out of memory
List<CubeSegmentScanner> scanners = Lists.newArrayList();
for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
@@ -229,9 +230,22 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return resultD;
}
- public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD, boolean isExactAggregation) {
- logger.info("Set isNeedStorageAggregation to " + !isExactAggregation);
- return !isExactAggregation;
+ public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) {
+
+ logger.info("GroupD :" + groupD);
+ logger.info("SingleValueD :" + singleValueD);
+ logger.info("Cuboid columns :" + cuboid.getColumns());
+
+ HashSet<TblColRef> temp = Sets.newHashSet();
+ temp.addAll(groupD);
+ temp.addAll(singleValueD);
+ if (cuboid.getColumns().size() == temp.size()) {
+ logger.info("Does not need storage aggregation");
+ return false;
+ } else {
+ logger.info("Need storage aggregation");
+ return true;
+ }
}
//exact aggregation was introduced back when we had some measures (like holistic distinct count) that is sensitive
@@ -268,7 +282,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
if (exact) {
- logger.info("exactAggregation is true");
+ logger.info("exactAggregation is true, cuboid id is " + cuboid.getId());
}
return exact;
}
@@ -355,7 +369,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
- private void setThreshold(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, StorageContext context) {
+ private void setThresholdIfNecessary(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, StorageContext context) {
boolean hasMemHungryMeasure = false;
for (FunctionDesc func : metrics) {
hasMemHungryMeasure |= func.getMeasureType().isMemoryHungry();
@@ -381,7 +395,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
- private void enableStoragePushDownLimit(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) {
+ private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) {
boolean possible = true;
boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
[33/35] kylin git commit: minor,
better log on dictionary creation failure
Posted by li...@apache.org.
minor, better log on dictionary creation failure
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6db4b172
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6db4b172
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6db4b172
Branch: refs/heads/1.5.x-HBase1.x
Commit: 6db4b1723c47762280128ec7c08a160fdc9a69d7
Parents: aef7869
Author: Li Yang <li...@apache.org>
Authored: Mon Sep 12 15:28:58 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Sep 12 15:28:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 2 +-
.../apache/kylin/dict/DictionaryGenerator.java | 378 +++++++++----------
.../apache/kylin/dict/DictionaryManager.java | 2 +
3 files changed, 192 insertions(+), 190 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6db4b172/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index de9051c..79ee084 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -449,7 +449,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getDictionaryMaxCardinality() {
- return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
+ return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "30000000"));
}
public int getTableSnapshotMaxMB() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6db4b172/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 5bd3357..ba848c6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -1,189 +1,189 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class DictionaryGenerator {
-
- private static final int DICT_MAX_CARDINALITY = getDictionaryMaxCardinality();
-
- private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
-
- private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
-
- private static int getDictionaryMaxCardinality() {
- try {
- return KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
- } catch (Throwable e) {
- return 5000000; // some test case does not have KylinConfig setup properly
- }
- }
-
- public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
- Preconditions.checkNotNull(dataType, "dataType cannot be null");
-
- // build dict, case by data type
- IDictionaryBuilder builder;
- if (dataType.isDateTimeFamily()) {
- if (dataType.isDate())
- builder = new DateDictBuilder();
- else
- builder = new TimeDictBuilder();
- } else if (dataType.isNumberFamily()) {
- builder = new NumberDictBuilder();
- } else {
- builder = new StringDictBuilder();
- }
-
- return buildDictionary(builder, null, valueEnumerator);
- }
-
- public static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException {
- int baseId = 0; // always 0 for now
- int nSamples = 5;
- ArrayList<String> samples = new ArrayList<String>(nSamples);
-
- Dictionary<String> dict = builder.build(dictInfo, valueEnumerator, baseId, nSamples, samples);
-
- // log a few samples
- StringBuilder buf = new StringBuilder();
- for (String s : samples) {
- if (buf.length() > 0) {
- buf.append(", ");
- }
- buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
- }
- logger.debug("Dictionary value samples: " + buf.toString());
- logger.debug("Dictionary cardinality: " + dict.getSize());
- logger.debug("Dictionary builder class: " + builder.getClass().getName());
- logger.debug("Dictionary class: " + dict.getClass().getName());
- if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
- throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- cardinality: " + dict.getSize());
- }
- return dict;
- }
-
- public static Dictionary mergeDictionaries(DataType dataType, List<DictionaryInfo> sourceDicts) throws IOException {
- return buildDictionary(dataType, new MultipleDictionaryValueEnumerator(sourceDicts));
- }
-
- private static class DateDictBuilder implements IDictionaryBuilder {
- @Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- final int BAD_THRESHOLD = 0;
- String matchPattern = null;
- byte[] value;
-
- for (String ptn : DATE_PATTERNS) {
- matchPattern = ptn; // be optimistic
- int badCount = 0;
- SimpleDateFormat sdf = new SimpleDateFormat(ptn);
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null || value.length == 0)
- continue;
-
- String str = Bytes.toString(value);
- try {
- sdf.parse(str);
- if (returnSamples.size() < nSamples && returnSamples.contains(str) == false)
- returnSamples.add(str);
- } catch (ParseException e) {
- logger.info("Unrecognized date value: " + str);
- badCount++;
- if (badCount > BAD_THRESHOLD) {
- matchPattern = null;
- break;
- }
- }
- }
- if (matchPattern != null) {
- return new DateStrDictionary(matchPattern, baseId);
- }
- }
-
- throw new IllegalStateException("Unrecognized datetime value");
- }
- }
-
- private static class TimeDictBuilder implements IDictionaryBuilder {
- @Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- return new TimeStrDictionary(); // base ID is always 0
- }
- }
-
- private static class StringDictBuilder implements IDictionaryBuilder {
- @Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter());
- byte[] value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null)
- continue;
- String v = Bytes.toString(value);
- builder.addValue(v);
- if (returnSamples.size() < nSamples && returnSamples.contains(v) == false)
- returnSamples.add(v);
- }
- return builder.build(baseId);
- }
- }
-
- private static class NumberDictBuilder implements IDictionaryBuilder {
- @Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter());
- byte[] value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null)
- continue;
- String v = Bytes.toString(value);
- if (StringUtils.isBlank(v)) // empty string is null for numbers
- continue;
-
- builder.addValue(v);
- if (returnSamples.size() < nSamples && returnSamples.contains(v) == false)
- returnSamples.add(v);
- }
- return builder.build(baseId);
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * @author yangli9
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class DictionaryGenerator {
+
+ private static final int DICT_MAX_CARDINALITY = getDictionaryMaxCardinality();
+
+ private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
+
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
+
+ private static int getDictionaryMaxCardinality() {
+ try {
+ return KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
+ } catch (Throwable e) {
+ return 30000000; // some test case does not have KylinConfig setup properly
+ }
+ }
+
+ public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ Preconditions.checkNotNull(dataType, "dataType cannot be null");
+
+ // build dict, case by data type
+ IDictionaryBuilder builder;
+ if (dataType.isDateTimeFamily()) {
+ if (dataType.isDate())
+ builder = new DateDictBuilder();
+ else
+ builder = new TimeDictBuilder();
+ } else if (dataType.isNumberFamily()) {
+ builder = new NumberDictBuilder();
+ } else {
+ builder = new StringDictBuilder();
+ }
+
+ return buildDictionary(builder, null, valueEnumerator);
+ }
+
+ public static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ int baseId = 0; // always 0 for now
+ int nSamples = 5;
+ ArrayList<String> samples = new ArrayList<String>(nSamples);
+
+ Dictionary<String> dict = builder.build(dictInfo, valueEnumerator, baseId, nSamples, samples);
+
+ // log a few samples
+ StringBuilder buf = new StringBuilder();
+ for (String s : samples) {
+ if (buf.length() > 0) {
+ buf.append(", ");
+ }
+ buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
+ }
+ logger.debug("Dictionary value samples: " + buf.toString());
+ logger.debug("Dictionary cardinality: " + dict.getSize());
+ logger.debug("Dictionary builder class: " + builder.getClass().getName());
+ logger.debug("Dictionary class: " + dict.getClass().getName());
+ if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
+ throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- cardinality: " + dict.getSize());
+ }
+ return dict;
+ }
+
+ public static Dictionary mergeDictionaries(DataType dataType, List<DictionaryInfo> sourceDicts) throws IOException {
+ return buildDictionary(dataType, new MultipleDictionaryValueEnumerator(sourceDicts));
+ }
+
+ private static class DateDictBuilder implements IDictionaryBuilder {
+ @Override
+ public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ final int BAD_THRESHOLD = 0;
+ String matchPattern = null;
+ byte[] value;
+
+ for (String ptn : DATE_PATTERNS) {
+ matchPattern = ptn; // be optimistic
+ int badCount = 0;
+ SimpleDateFormat sdf = new SimpleDateFormat(ptn);
+ while (valueEnumerator.moveNext()) {
+ value = valueEnumerator.current();
+ if (value == null || value.length == 0)
+ continue;
+
+ String str = Bytes.toString(value);
+ try {
+ sdf.parse(str);
+ if (returnSamples.size() < nSamples && returnSamples.contains(str) == false)
+ returnSamples.add(str);
+ } catch (ParseException e) {
+ logger.info("Unrecognized date value: " + str);
+ badCount++;
+ if (badCount > BAD_THRESHOLD) {
+ matchPattern = null;
+ break;
+ }
+ }
+ }
+ if (matchPattern != null) {
+ return new DateStrDictionary(matchPattern, baseId);
+ }
+ }
+
+ throw new IllegalStateException("Unrecognized datetime value");
+ }
+ }
+
+ private static class TimeDictBuilder implements IDictionaryBuilder {
+ @Override
+ public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ return new TimeStrDictionary(); // base ID is always 0
+ }
+ }
+
+ private static class StringDictBuilder implements IDictionaryBuilder {
+ @Override
+ public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter());
+ byte[] value;
+ while (valueEnumerator.moveNext()) {
+ value = valueEnumerator.current();
+ if (value == null)
+ continue;
+ String v = Bytes.toString(value);
+ builder.addValue(v);
+ if (returnSamples.size() < nSamples && returnSamples.contains(v) == false)
+ returnSamples.add(v);
+ }
+ return builder.build(baseId);
+ }
+ }
+
+ private static class NumberDictBuilder implements IDictionaryBuilder {
+ @Override
+ public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ byte[] value;
+ while (valueEnumerator.moveNext()) {
+ value = valueEnumerator.current();
+ if (value == null)
+ continue;
+ String v = Bytes.toString(value);
+ if (StringUtils.isBlank(v)) // empty string is null for numbers
+ continue;
+
+ builder.addValue(v);
+ if (returnSamples.size() < nSamples && returnSamples.contains(v) == false)
+ returnSamples.add(v);
+ }
+ return builder.build(baseId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6db4b172/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index ff088b9..c8a7a54 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -321,6 +321,8 @@ public class DictionaryManager {
dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()), columnValueEnumerator);
else
dictionary = DictionaryGenerator.buildDictionary((IDictionaryBuilder) ClassUtil.newInstance(builderClass), dictInfo, columnValueEnumerator);
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to create dictionary on " + col, ex);
} finally {
if (columnValueEnumerator != null)
columnValueEnumerator.close();
[23/35] kylin git commit: minor, fix job start/end in diagnosis
Posted by li...@apache.org.
minor, fix job start/end in diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7cbf673
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7cbf673
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7cbf673
Branch: refs/heads/1.5.x-HBase1.x
Commit: d7cbf6732a9571007de61dd492ed4d7559cbf9ac
Parents: d7a3fdf
Author: lidongsjtu <li...@apache.org>
Authored: Sat Sep 10 15:23:43 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sat Sep 10 16:09:19 2016 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7cbf673/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
index 086a84f..ef77c6a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
@@ -133,6 +133,8 @@ public class JobInstanceExtractor extends AbstractInfoExtractor {
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setExecStartTime(AbstractExecutable.getStartTime(output));
+ result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output)) / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
[14/35] kylin git commit: KYLIN-1922 optimize needStorageAggregation
check logic and make sure self-termination in coprocessor works
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
new file mode 100644
index 0000000..0cdfa7e
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.gridtable.CubeCodeSystem;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.GTBuilder;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.GridTable;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
+import org.apache.kylin.gridtable.GTInfo.Builder;
+import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class DictGridTableTest extends LocalFileMetadataTestCase {
+
+ private GridTable table;
+ private GTInfo info;
+ private CompareTupleFilter timeComp0;
+ private CompareTupleFilter timeComp1;
+ private CompareTupleFilter timeComp2;
+ private CompareTupleFilter timeComp3;
+ private CompareTupleFilter timeComp4;
+ private CompareTupleFilter timeComp5;
+ private CompareTupleFilter timeComp6;
+ private CompareTupleFilter ageComp1;
+ private CompareTupleFilter ageComp2;
+ private CompareTupleFilter ageComp3;
+ private CompareTupleFilter ageComp4;
+
+ @After
+ public void after() throws Exception {
+
+ this.cleanupTestMetadata();
+ }
+
+ @Before
+ public void setup() throws IOException {
+
+ this.createTestMetadata();
+
+ table = newTestTable();
+ info = table.getInfo();
+
+ timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14"));
+ timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
+ timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
+ timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
+ timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15"));
+ timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14"));
+ ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
+ ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
+ ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
+ ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
+
+ }
+
+ @Test
+ public void verifySegmentSkipping() {
+
+ ByteArray segmentStart = enc(info, 0, "2015-01-14");
+ ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+ assertEquals(segmentStart, segmentStartX);
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp2, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp4, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
+ assertEquals("[[null, 10, null, null, null], [1421193600000, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
+ assertEquals(0, r.get(0).fuzzyKeys.size());
+ }
+ {
+ //skip FALSE filter
+ LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ //TRUE or FALSE filter
+ LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[null, null]-[null, null]", r.get(0).toString());
+ }
+ {
+ //TRUE or other filter
+ LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[null, null]-[null, null]", r.get(0).toString());
+ }
+ }
+
+ @Test
+ public void verifySegmentSkipping2() {
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());//scan range are [close,close]
+ }
+ }
+
+ @Test
+ public void verifyScanRangePlanner() {
+
+ // flatten or-and & hbase fuzzy value
+ {
+ LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
+ assertEquals("[[null, 10, null, null, null], [null, 20, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+
+ // pre-evaluate ever false
+ {
+ LogicalTupleFilter filter = and(timeComp1, timeComp2);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+
+ // pre-evaluate ever true
+ {
+ LogicalTupleFilter filter = or(timeComp1, ageComp4);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals("[[null, null]-[null, null]]", r.toString());
+ }
+
+ // merge overlap range
+ {
+ LogicalTupleFilter filter = or(timeComp1, timeComp3);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals("[[null, null]-[null, null]]", r.toString());
+ }
+
+ // merge too many ranges
+ {
+ LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(3, r.size());
+ assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
+ assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString());
+ assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString());
+ planner.setMaxScanRanges(2);
+ List<GTScanRange> r2 = planner.planScanRanges();
+ assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString());
+ }
+ }
+
+ @Test
+ public void verifyFirstRow() throws IOException {
+ doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", //
+ "[1421193600000, 30, Luke, 10, 10.5]", //
+ "[1421280000000, 20, Dong, 10, 10.5]", //
+ "[1421280000000, 20, Jason, 10, 10.5]", //
+ "[1421280000000, 30, Xu, 10, 10.5]", //
+ "[1421366400000, 20, Mahone, 10, 10.5]", //
+ "[1421366400000, 20, Qianhao, 10, 10.5]", //
+ "[1421366400000, 30, George, 10, 10.5]", //
+ "[1421366400000, 30, Shaofeng, 10, 10.5]", //
+ "[1421452800000, 10, Kejia, 10, 10.5]");
+ }
+
+ //for testing GTScanRequest serialization and deserialization
+ public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
+ ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
+ GTScanRequest.serializer.serialize(origin, buffer);
+ buffer.flip();
+ GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer);
+
+ Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs());
+ Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01);
+ return sGTScanRequest;
+ }
+
+ @Test
+ public void verifyScanWithUnevaluatableFilter() throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
+ LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
+ LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);
+
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }).setFilterPushDown(filter).createGTScanRequest();
+
+ // note the unEvaluatable column 1 in filter is added to group by
+ assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
+ }
+
+ @Test
+ public void verifyScanWithEvaluatableFilter() throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }).setFilterPushDown(filter).createGTScanRequest();
+ // note the evaluatable column 1 in filter is added to returned columns but not in group by
+ assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]");
+ }
+
+ @Test
+ public void testFilterScannerPerf() throws IOException {
+ GridTable table = newTestPerfTable();
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ FilterResultCache.ENABLED = false;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = true;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = false;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = true;
+ testFilterScannerPerfInner(table, info, filter);
+ }
+
+ @SuppressWarnings("unused")
+ private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException {
+ long start = System.currentTimeMillis();
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(filter).createGTScanRequest();
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ i++;
+ }
+ scanner.close();
+ long end = System.currentTimeMillis();
+ System.out.println((end - start) + "ms with filter cache enabled=" + FilterResultCache.ENABLED + ", " + i + " rows");
+ }
+
+ @Test
+ public void verifyConvertFilterConstants1() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants2() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1<"9" round up to $1<"10"
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants3() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1<="9" round down to FALSE
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants4() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1 in ("9", "10", "15") has only "10" left
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString());
+ }
+
+ private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
+ System.out.println(req);
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ System.out.println(r);
+ if (verifyRows == null || i >= verifyRows.length) {
+ Assert.fail();
+ }
+ assertEquals(verifyRows[i], r.toString());
+ i++;
+ }
+ scanner.close();
+ }
+
+ public static ByteArray enc(GTInfo info, int col, String value) {
+ ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
+ info.getCodeSystem().encodeColumnValue(col, value, buf);
+ return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
+ }
+
+ public static ExtractTupleFilter unevaluatable(TblColRef col) {
+ ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
+ r.addChild(new ColumnTupleFilter(col));
+ return r;
+ }
+
+ public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
+ CompareTupleFilter result = new CompareTupleFilter(op);
+ result.addChild(new ColumnTupleFilter(col));
+ result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
+ return result;
+ }
+
+ public static LogicalTupleFilter and(TupleFilter... children) {
+ return logic(FilterOperatorEnum.AND, children);
+ }
+
+ public static LogicalTupleFilter or(TupleFilter... children) {
+ return logic(FilterOperatorEnum.OR, children);
+ }
+
+ public static LogicalTupleFilter not(TupleFilter child) {
+ return logic(FilterOperatorEnum.NOT, child);
+ }
+
+ public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+ LogicalTupleFilter result = new LogicalTupleFilter(op);
+ for (TupleFilter c : children) {
+ result.addChild(c);
+ }
+ return result;
+ }
+
+ public static GridTable newTestTable() throws IOException {
+ GTInfo info = newInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.close();
+
+ return table;
+ }
+
+ static GridTable newTestPerfTable() throws IOException {
+ GTInfo info = newInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ for (int i = 0; i < 100000; i++) {
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ }
+ builder.close();
+
+ return table;
+ }
+
+ static GTInfo newInfo() {
+ Builder builder = GTInfo.builder();
+ builder.setCodeSystem(newDictCodeSystem());
+ builder.setColumns( //
+ DataType.getType("timestamp"), //
+ DataType.getType("integer"), //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("bigint"), //
+ DataType.getType("decimal") //
+ );
+ builder.setPrimaryKey(setOf(0, 1));
+ builder.setColumnPreferIndex(setOf(0));
+ builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
+ builder.enableRowBlock(4);
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static CubeCodeSystem newDictCodeSystem() {
+ DimensionEncoding[] dimEncs = new DimensionEncoding[3];
+ dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
+ dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString());
+ return new CubeCodeSystem(dimEncs);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfString() {
+ TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("Dong");
+ builder.addValue("George");
+ builder.addValue("Jason");
+ builder.addValue("Kejia");
+ builder.addValue("Luke");
+ builder.addValue("Mahone");
+ builder.addValue("Qianhao");
+ builder.addValue("Shaofeng");
+ builder.addValue("Xu");
+ builder.addValue("Yang");
+ return builder.build(0);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfInteger() {
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("10");
+ builder.addValue("20");
+ builder.addValue("30");
+ builder.addValue("40");
+ builder.addValue("50");
+ builder.addValue("60");
+ builder.addValue("70");
+ builder.addValue("80");
+ builder.addValue("90");
+ builder.addValue("100");
+ return builder.build(0);
+ }
+
+ public static ImmutableBitSet setOf(int... values) {
+ BitSet set = new BitSet();
+ for (int i : values)
+ set.set(i);
+ return new ImmutableBitSet(set);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 375b198..fc2fd52 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -22,13 +22,16 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
@@ -37,18 +40,26 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
public class ITKylinQueryTest extends KylinTestBase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@BeforeClass
public static void setUp() throws Exception {
printInfo("setUp in ITKylinQueryTest");
@@ -108,10 +119,52 @@ public class ITKylinQueryTest extends KylinTestBase {
return "";
}
- @Ignore("this is only for debug")
+ @Test
+ public void testTimeoutQuery() throws Exception {
+
+ thrown.expect(SQLException.class);
+
+ //should not break at table duplicate check, should fail at model duplicate check
+ thrown.expectCause(new BaseMatcher<Throwable>() {
+ @Override
+ public boolean matches(Object item) {
+ if (item instanceof GTScanSelfTerminatedException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ }
+ });
+
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ BackdoorToggles.setToggles(toggles);
+
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
+
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
+ BackdoorToggles.cleanToggles();
+ }
+
+ //don't try to ignore this test, try to clean your "temp" folder
@Test
public void testTempQuery() throws Exception {
+ PRINT_RESULT = true;
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/temp", null, true);
+ PRINT_RESULT = false;
}
@Ignore
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/kylin-it/src/test/resources/query/sql_timeout/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
new file mode 100644
index 0000000..3b9a837
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 1200
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
index 75533cd..5f21351 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
@@ -26,4 +26,5 @@ public enum CoprocessorBehavior {
SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
SCAN_FILTER_AGGR, //aggregate the result. Will return results
SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 07a3cc3..5b48351 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -43,8 +43,8 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTScanRange;
import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -106,7 +106,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
- logger.debug("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
+ logger.info("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard();
short shardNum = shardNumAndBaseShard.getFirst();
@@ -146,7 +146,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
rawScanBufferSize *= 4;
}
}
- scanRequest.setGTScanRanges(Lists.<GTScanRange> newArrayList());//since raw scans are sent to coprocessor, we don't need to duplicate sending it
+ scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
@@ -248,7 +248,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
if (abnormalFinish[0]) {
- Throwable ex = new RuntimeException(logHeader + "The coprocessor thread stopped itself due to scan timeout or scan threshold(check region server log), failing current query...");
+ Throwable ex = new GTScanSelfTerminatedException(logHeader + "The coprocessor thread stopped itself due to scan timeout or scan threshold(check region server log), failing current query...");
logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout
epResultItr.notifyCoprocException(ex);
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index a359d19..f1e5dab 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -213,7 +213,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize());
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize(), false);
IGTScanner rawScanner = store.scan(scanRequest);
final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 7d48c1a..442963f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -18,20 +18,22 @@
package org.apache.kylin.storage.hbase.cube.v2;
+import java.util.Iterator;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
class ExpectedSizeIterator implements Iterator<byte[]> {
private static final Logger logger = LoggerFactory.getLogger(ExpectedSizeIterator.class);
@@ -48,22 +50,24 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
this.expectedSize = expectedSize;
this.queue = new ArrayBlockingQueue<byte[]>(expectedSize);
+ StringBuilder sb = new StringBuilder();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+
this.rpcTimeout = hconf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
this.timeout = this.rpcTimeout * hconf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
- logger.info("rpc timeout is {} and after multiply retry times become {}", this.rpcTimeout, this.timeout);
- this.timeout = Math.max(this.timeout, 5 * 60000);
+ sb.append("rpc timeout is " + this.rpcTimeout + " and after multiply retry times becomes " + this.timeout);
+
this.timeout *= KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
+ sb.append(" after multiply kylin.query.cube.visit.timeout.times becomes " + this.timeout);
+
+ logger.info(sb.toString());
if (BackdoorToggles.getQueryTimeout() != -1) {
this.timeout = BackdoorToggles.getQueryTimeout();
+ logger.info("rpc timeout is overwritten to " + this.timeout);
}
- this.timeout *= 1.1; // allow for some delay
-
- logger.info("Final Timeout for ExpectedSizeIterator is: " + this.timeout);
-
- this.timeoutTS = System.currentTimeMillis() + this.timeout;
+ this.timeoutTS = System.currentTimeMillis() + 2 * this.timeout;//longer timeout than coprocessor so that query thread will not timeout faster than coprocessor
}
@Override
@@ -85,9 +89,14 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
}
if (coprocException != null) {
- throw new RuntimeException("Error in coprocessor", coprocException);
+ if (coprocException instanceof GTScanSelfTerminatedException)
+ throw (GTScanSelfTerminatedException) coprocException;
+ else
+ throw new RuntimeException("Error in coprocessor",coprocException);
+
} else if (ret == null) {
- throw new RuntimeException("Timeout visiting cube!");
+ throw new RuntimeException("Timeout visiting cube! Check why coprocessor exception is not sent back? In coprocessor Self-termination is checked every " + //
+ GTScanRequest.terminateCheckInterval + " scanned rows, the configured timeout(" + timeout + ") cannot support this many scans?");
} else {
return ret;
}
@@ -110,7 +119,7 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
}
public long getRpcTimeout() {
- return this.rpcTimeout;
+ return this.timeout;
}
public void notifyCoprocException(Throwable ex) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 4b9b4fa..1d8ad79 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -43,13 +43,15 @@ public class HBaseReadonlyStore implements IGTStore {
private List<Pair<byte[], byte[]>> hbaseColumns;
private List<List<Integer>> hbaseColumnsToGT;
private int rowkeyPreambleSize;
+ private boolean withDelay = false;
- public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize) {
+ public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize, boolean withDelay) {
this.cellListIterator = cellListIterator;
this.info = gtScanRequest.getInfo();
this.hbaseColumns = hbaseColumns;
this.hbaseColumnsToGT = hbaseColumnsToGT;
this.rowkeyPreambleSize = rowkeyPreambleSize;
+ this.withDelay = withDelay;
}
@Override
@@ -95,6 +97,13 @@ public class HBaseReadonlyStore implements IGTStore {
@Override
public boolean hasNext() {
+ if (withDelay) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
return cellListIterator.hasNext();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index b29d0d1..064d100 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -170,7 +170,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
@SuppressWarnings("checkstyle:methodlength")
@Override
- public void visitCube(final RpcController controller, CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
+ public void visitCube(final RpcController controller, final CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
List<RegionScanner> regionScanners = Lists.newArrayList();
HRegion region = null;
@@ -241,7 +241,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
if (behavior.ordinal() < CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
- scanReq.setAggCacheMemThreshold(0); // disable mem check if so told
+ scanReq.disableAggCacheMemCheck(); // disable mem check if so told
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
@@ -266,7 +266,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter);
}
- if (counter % 100000 == 1) {
+ if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
logger.info("Scanned " + counter + " rows from HBase.");
}
counter++;
@@ -284,7 +284,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize());
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, //
+ request.getRowkeyPreambleSize(), CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(request.getBehavior()));
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, //
@@ -299,14 +300,9 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
for (GTRecord oneRecord : finalScanner) {
- if (finalRowCount > storagePushDownLimit) {
- logger.info("The finalScanner aborted because storagePushDownLimit is satisfied");
- break;
- }
-
- if (finalRowCount % 100000 == 1) {
+ if (finalRowCount % GTScanRequest.terminateCheckInterval == 1) {
if (System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("finalScanner timeouts after scanned " + finalRowCount);
+ throw new GTScanTimeoutException("finalScanner timeouts after contributed " + finalRowCount);
}
}
@@ -319,7 +315,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
outputStream.write(buffer.array(), 0, buffer.position());
+
finalRowCount++;
+
+ //if it's doing storage aggr, then should rely on GTAggregateScanner's limit check
+ if (!scanReq.isDoingStorageAggregation() && finalRowCount >= storagePushDownLimit) {
+ //read one more record than limit
+ logger.info("The finalScanner aborted because storagePushDownLimit is satisfied");
+ break;
+ }
}
} catch (GTScanTimeoutException e) {
scanNormalComplete.setValue(false);