You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/04 04:43:30 UTC
[01/47] kylin git commit: KYLIN-2348 let choose model freely for
sub-queries [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2361 f577a7da0 -> 9a3bd71c8 (forced update)
KYLIN-2348 let choose model freely for sub-queries
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d52bb8bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d52bb8bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d52bb8bc
Branch: refs/heads/KYLIN-2361
Commit: d52bb8bc05bd530362fe0970b8e8a85f8b2289bc
Parents: 51c6571
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 18 22:46:24 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 18 22:46:24 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/query/relnode/OLAPTableScan.java | 2 ++
.../apache/kylin/query/routing/ModelChooser.java | 16 ++++++++--------
2 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 452170a..f7877be 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -53,6 +53,7 @@ import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
import org.apache.calcite.rel.rules.JoinUnionTransposeRule;
import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SemiJoinRule;
import org.apache.calcite.rel.rules.SortJoinTransposeRule;
import org.apache.calcite.rel.rules.SortUnionTransposeRule;
import org.apache.calcite.rel.type.RelDataType;
@@ -174,6 +175,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
planner.removeRule(DateRangeRules.FILTER_INSTANCE);
+ planner.removeRule(SemiJoinRule.INSTANCE);
// distinct count will be split into a separated query that is joined with the left query
planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 2517dc5..f979f7b 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -53,17 +53,17 @@ public class ModelChooser {
IdentityHashMap<OLAPContext, Set<IRealization>> candidates = new IdentityHashMap<>();
// attempt one model for all contexts
- Set<IRealization> reals = attemptSelectModel(contexts);
- if (reals != null) {
- for (OLAPContext ctx : contexts) {
- candidates.put(ctx, reals);
- }
- return candidates;
- }
+ // Set<IRealization> reals = attemptSelectModel(contexts);
+ // if (reals != null) {
+ // for (OLAPContext ctx : contexts) {
+ // candidates.put(ctx, reals);
+ // }
+ // return candidates;
+ // }
// try different model for different context
for (OLAPContext ctx : contexts) {
- reals = attemptSelectModel(ImmutableList.of(ctx));
+ Set<IRealization> reals = attemptSelectModel(ImmutableList.of(ctx));
if (reals == null)
throw new NoRealizationFoundException("No model found for" + toErrorMsg(ctx));
[09/47] kylin git commit: KYLIN-2411 Kill MR job on pause
Posted by bi...@apache.org.
KYLIN-2411 Kill MR job on pause
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b49c9e39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b49c9e39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b49c9e39
Branch: refs/heads/KYLIN-2361
Commit: b49c9e3951c29a21e71f179ee77e2295fbc52ba9
Parents: 38c3e7b
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 19 11:02:12 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 19 17:37:42 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/execution/ExecutableManager.java | 2 +-
.../apache/kylin/engine/mr/common/MapReduceExecutable.java | 8 ++++++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 466cdad..48cedb5 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -347,7 +347,7 @@ public class ExecutableManager {
for (AbstractExecutable task : tasks) {
if (task.getId().compareTo(stepId) >= 0) {
logger.debug("rollback task : " + task);
- updateJobOutput(task.getId(), ExecutableState.READY, null, null);
+ updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String>newHashMap(), "");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index f887c4c..6de07ca 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -148,7 +148,7 @@ public class MapReduceExecutable extends AbstractExecutable {
// boolean useKerberosAuth = context.getConfig().isGetJobStatusWithKerberos();
// HadoopStatusChecker statusChecker = new HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
JobStepStatusEnum status = JobStepStatusEnum.NEW;
- while (!isDiscarded()) {
+ while (!isDiscarded() && !isPaused()) {
JobStepStatusEnum newStatus = HadoopJobStatusChecker.checkStatus(job, output);
if (status == JobStepStatusEnum.KILLED) {
@@ -184,7 +184,11 @@ public class MapReduceExecutable extends AbstractExecutable {
}
}
- return new ExecuteResult(ExecuteResult.State.DISCARDED, output.toString());
+ if (isDiscarded()) {
+ return new ExecuteResult(ExecuteResult.State.DISCARDED, output.toString());
+ } else {
+ return new ExecuteResult(ExecuteResult.State.STOPPED, output.toString());
+ }
} catch (ReflectiveOperationException e) {
logger.error("error getMapReduceJobClass, class name:" + getParam(KEY_MR_JOB), e);
[13/47] kylin git commit: KYLIN 2370 Refine unload and reload table
Posted by bi...@apache.org.
KYLIN 2370 Refine unload and reload table
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a853a7c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a853a7c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a853a7c4
Branch: refs/heads/KYLIN-2361
Commit: a853a7c4b6b7d42cc3ffd066b618f5ea72ca0d9f
Parents: 85a1eb3
Author: chenzhx <34...@qq.com>
Authored: Mon Jan 9 15:09:16 2017 +0800
Committer: zhongjian <ji...@163.com>
Committed: Fri Jan 20 16:21:11 2017 +0800
----------------------------------------------------------------------
webapp/app/css/AdminLTE.css | 27 +++
webapp/app/js/controllers/sourceMeta.js | 175 ++++++++-----------
.../js/directives/kylin_abn_tree_directive.js | 7 +-
.../app/partials/tables/source_table_tree.html | 6 +-
webapp/app/partials/tables/table_unload.html | 33 ----
5 files changed, 111 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/css/AdminLTE.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 857dbf7..c7740d1 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4800,3 +4800,30 @@ Gradient Background colors
white-space: normal !important;
}
}
+
+.abn-tree .abn-tree-row .tree-table-btn{
+ width: 4%;
+ float: right;
+ border-radius: 5px;
+ margin-right: 5px;
+ padding: 3px 16px 2px 5px;
+ color: #ffffff;
+ position:static;
+}
+.abn-tree .abn-tree-row .tree-table-btn .tooltip{
+ overflow:visible;
+}
+.abn-tree .abn-tree-row .btn-info:hover{
+ background-color:#269abc;
+}
+.abn-tree > .abn-tree-row.active > .btn-info{
+ background-color:#269abc;
+ border-left-color:#269abc;
+}
+.abn-tree .abn-tree-row .btn-success:hover{
+ background-color:#008d4c;
+}
+.abn-tree > .abn-tree-row.active > .btn-success{
+ background-color:#008d4c;
+ border-left-color:#008d4c;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index a53a35f..5421673 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,31 +132,83 @@ KylinApp
});
};
- $scope.openUnLoadModal = function () {
- if(!$scope.projectModel.selectedProject){
- SweetAlert.swal('Oops...', "Please select a project.", 'info');
+ $scope.reloadTable = function (tableName,projectName){
+ loadingRequest.show();
+ TableService.loadHiveTable({tableName: tableName, action: projectName}, {calculate: $scope.isCalculate}, function (result) {
+ var loadTableInfo = "";
+ angular.forEach(result['result.loaded'], function (table) {
+ loadTableInfo += "\n" + table;
+ })
+ var unloadedTableInfo = "";
+ angular.forEach(result['result.unloaded'], function (table) {
+ unloadedTableInfo += "\n" + table;
+ })
+ if (result['result.unloaded'].length != 0 && result['result.loaded'].length == 0) {
+ SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo, 'error');
+ }
+ if (result['result.loaded'].length != 0 && result['result.unloaded'].length == 0) {
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded: ' + loadTableInfo, 'success');
+ }
+ if (result['result.loaded'].length != 0 && result['result.unloaded'].length != 0) {
+ SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo, 'warning');
+ }
+ loadingRequest.hide();
+ $scope.aceSrcTbLoaded(true);
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ loadingRequest.hide();
+ })
+ }
+
+
+
+ $scope.removeList = function (tableName,projectName) {
+ if (tableName.trim() === "") {
+ SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info');
return;
}
- $modal.open({
- templateUrl: 'removeHiveTable.html',
- controller: ModalInstanceCtrl,
- backdrop : 'static',
- resolve: {
- tableNames: function () {
- return $scope.tableNames;
- },
- projectName: function () {
- return $scope.projectModel.selectedProject;
- },
- isCalculate: function () {
- return $scope.isCalculate;
- },
- scope: function () {
- return $scope;
- }
+ if (!projectName) {
+ SweetAlert.swal('', 'Please choose your project first!.', 'info');
+ return;
+ }
+ loadingRequest.show();
+ TableService.unLoadHiveTable({tableName: tableName, action: projectName}, {}, function (result) {
+ var removedTableInfo = "";
+ angular.forEach(result['result.unload.success'], function (table) {
+ removedTableInfo += "\n" + table;
+ })
+ var unRemovedTableInfo = "";
+ angular.forEach(result['result.unload.fail'], function (table) {
+ unRemovedTableInfo += "\n" + table;
+ })
+ if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) {
+ SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo, 'error');
+ }
+ if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) {
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo, 'success');
+ }
+ if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) {
+ SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo, 'warning');
+ }
+ loadingRequest.hide();
+ $scope.aceSrcTbLoaded(true);
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
}
- });
- };
+ loadingRequest.hide();
+ })
+ }
var ModalInstanceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, isCalculate, scope, kylinConfig) {
$scope.tableNames = "";
@@ -340,88 +392,11 @@ KylinApp
}
$scope.cancel();
- loadingRequest.show();
- TableService.loadHiveTable({tableName: $scope.tableNames, action: projectName}, {calculate: $scope.isCalculate}, function (result) {
- var loadTableInfo = "";
- angular.forEach(result['result.loaded'], function (table) {
- loadTableInfo += "\n" + table;
- })
- var unloadedTableInfo = "";
- angular.forEach(result['result.unloaded'], function (table) {
- unloadedTableInfo += "\n" + table;
- })
-
- if (result['result.unloaded'].length != 0 && result['result.loaded'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo, 'error');
- }
- if (result['result.loaded'].length != 0 && result['result.unloaded'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded: ' + loadTableInfo, 'success');
- }
- if (result['result.loaded'].length != 0 && result['result.unloaded'].length != 0) {
- SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo, 'warning');
- }
- loadingRequest.hide();
- scope.aceSrcTbLoaded(true);
-
- }, function (e) {
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- loadingRequest.hide();
- })
+ scope.reloadTable ($scope.tableNames,projectName);
}
- $scope.remove = function () {
- if ($scope.tableNames.trim() === "") {
- SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info');
- return;
- }
-
- if (!$scope.projectName) {
- SweetAlert.swal('', 'Please choose your project first!.', 'info');
- return;
- }
-
- $scope.cancel();
- loadingRequest.show();
- TableService.unLoadHiveTable({tableName: $scope.tableNames, action: projectName}, {}, function (result) {
- var removedTableInfo = "";
- angular.forEach(result['result.unload.success'], function (table) {
- removedTableInfo += "\n" + table;
- })
- var unRemovedTableInfo = "";
- angular.forEach(result['result.unload.fail'], function (table) {
- unRemovedTableInfo += "\n" + table;
- })
-
- if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo, 'error');
- }
- if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo, 'success');
- }
- if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) {
- SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo, 'warning');
- }
- loadingRequest.hide();
- scope.aceSrcTbLoaded(true);
- }, function (e) {
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- loadingRequest.hide();
- })
- }
};
$scope.editStreamingConfig = function(tableName){
http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/directives/kylin_abn_tree_directive.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/kylin_abn_tree_directive.js b/webapp/app/js/directives/kylin_abn_tree_directive.js
index a3023ab..7545666 100644
--- a/webapp/app/js/directives/kylin_abn_tree_directive.js
+++ b/webapp/app/js/directives/kylin_abn_tree_directive.js
@@ -31,9 +31,13 @@
'$timeout', function($timeout) {
return {
restrict: 'E',
- template: "<ul class=\"nav nav-list nav-pills nav-stacked abn-tree\">\n <li ng-repeat=\"row in tree_rows | filter:{visible:true} track by row.branch.uid\" ng-animate=\"'abn-tree-animate'\" ng-class=\"'level-' + {{ row.level }} + (row.branch.selected ? ' active':'') + ' ' +row.classes.join(' ')\" class=\"abn-tree-row\"><a ng-click=\"user_clicks_branch(row.branch)\" ng-dblclick=\"user_dbClicks_branch(row.branch)\"><i ng-class=\"row.tree_icon\" class=\"indented tree-icon\"> </i><span class=\"indented tree-label\">{{ row.label }} </span></a></li>\n</ul>",
+ template: "<ul class=\"nav nav-list nav-pills nav-stacked abn-tree\">\n <li data=\"{{row.branch.fullName}}\" ng-repeat=\"row in tree_rows | filter:{visible:true} track by row.branch.uid\" ng-animate=\"'abn-tree-animate'\" ng-class=\"'level-' + {{ row.level }} + (row.branch.selected ? ' active':'') + ' ' +row.classes.join(' ')\" class=\"abn-tree-row\" ><a ng-click=\"user_clicks_branch(row.branch)\" ng-dblclick=\"user_dbClicks_branch(row.branch)\" style=\"width:80%;float:left;\"><i ng-class=\"row.tree_icon\" class=\"indented tree-icon\" > </i><span class=\"indented tree-label\">{{ row.label }} </span></a> <a class=\"btn btn-xs btn-info tree-table-btn\" ng-if=\"row.branch.data.exd&&row.level==2&&userService.hasRole('ROLE_ADMIN')&&row.branch.data.source_type==0 \" tooltip=\"UnLoad Hive Table\" tooltip-placement=\"left\" ng-click=\"unloadTable({tableName:row.branch.label,projectName:projectName})\" ><i class=\"fa fa-remove\"></i></a> <a class=\"btn btn-xs btn-success tree-ta
ble-btn\" tooltip-placement=\"left\" tooltip=\"ReLoad Hive Table\" ng-if=\"row.level==2&&userService.hasRole('ROLE_ADMIN')&&row.branch.data.source_type==0\" ng-click=\"reloadTable({tableName:row.branch.label,projectName:projectName})\"><i class=\"fa fa-download\"></i></a> </li>\n</ul>",
replace: true,
scope: {
+ userService:'=',
+ reloadTable:'&',
+ unloadTable:'&',
+ projectName:'@',
treeData: '=',
onSelect: '&',
onDblclick:'&',
@@ -186,7 +190,6 @@
return b.uid = "" + Math.random();
}
});
- console.log('UIDs are set.');
for_each_branch(function(b) {
var child, _i, _len, _ref, _results;
if (angular.isArray(b.children)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/partials/tables/source_table_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/source_table_tree.html b/webapp/app/partials/tables/source_table_tree.html
index 3008d8c..eb1ad79 100755
--- a/webapp/app/partials/tables/source_table_tree.html
+++ b/webapp/app/partials/tables/source_table_tree.html
@@ -27,7 +27,6 @@
<div class="pull-right">
<a class="btn btn-xs btn-primary" tooltip="Load Hive Table" ng-if="userService.hasRole('ROLE_ADMIN')" ng-click="openModal()"><i class="fa fa-download"></i></a>
<a class="btn btn-xs btn-info" tooltip="Load Hive Table From Tree" ng-if="userService.hasRole('ROLE_ADMIN')" ng-click="openTreeModal()"><i class="fa fa-download"></i></a>
- <a class="btn btn-xs btn-info" tooltip="Unload Hive Table" ng-if="userService.hasRole('ROLE_ADMIN')" ng-click="openUnLoadModal()"><i class="fa fa-remove"></i></a>
<a class="btn btn-xs btn-primary" tooltip="Add Streaming Table" ng-if="userService.hasRole('ROLE_ADMIN')" ng-click="openStreamingSourceModal()"><i class="fa fa-area-chart"></i></a>
</div>
</div>
@@ -37,6 +36,10 @@
<!--tree-->
<div style="width:100%; height:{{window}}px; overflow:auto;">
<abn-tree
+ user-service = "userService"
+ reload-table = "reloadTable(tableName,projectName)"
+ unload-table = "removeList(tableName,projectName)"
+ project-name = "{{projectModel.selectedProject}}"
tree-data = "tableModel.selectedSrcDb"
tree-control = "my_tree"
icon-leaf = "fa fa-columns"
@@ -75,4 +78,3 @@
</script>
<div ng-include="'partials/tables/table_load.html'"></div>
-<div ng-include="'partials/tables/table_unload.html'"></div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/partials/tables/table_unload.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/table_unload.html b/webapp/app/partials/tables/table_unload.html
deleted file mode 100644
index 258551e..0000000
--- a/webapp/app/partials/tables/table_unload.html
+++ /dev/null
@@ -1,33 +0,0 @@
-<!--
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
--->
-
- <script type="text/ng-template" id="removeHiveTable.html">
- <div class="modal-header">
- <h4>Unload Hive Table Metadata</h4>
- </div>
- <div class="modal-body">
- <span><strong>Project: </strong>{{ $parent.projectName!=null?$parent.projectName:'NULL'}}</span>
- <label for="tables"> Table Names:(Seperate with comma)</label>
- <textarea ng-model="$parent.tableNames" class="form-control" id="tables"
- placeholder="table1,table2 By default,system will choose 'Default' as database,you can specify database like this 'database.table'"></textarea>
- </div>
- <div class="modal-footer">
- <button class="btn btn-primary" ng-click="remove()">Unload</button>
- <button class="btn btn-primary" ng-click="cancel()">Cancel</button>
- </div>
- </script>
[43/47] kylin git commit: minor,
make DefaultSchedulerTest.testDiscard() more stable
Posted by bi...@apache.org.
minor, make DefaultSchedulerTest.testDiscard() more stable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0582512e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0582512e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0582512e
Branch: refs/heads/KYLIN-2361
Commit: 0582512ea06bcd1c662903ff69012ba1cc7cc33c
Parents: c3fff6d
Author: Li Yang <li...@apache.org>
Authored: Thu Jan 26 11:49:04 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Jan 26 11:49:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/job/SelfStopExecutable.java | 31 ++++++++++++++++----
.../impl/threadpool/DefaultSchedulerTest.java | 5 ++--
2 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index b4f6a98..9a3eb48 100644
--- a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -26,21 +26,40 @@ import org.apache.kylin.job.execution.ExecuteResult;
*/
public class SelfStopExecutable extends BaseTestExecutable {
+ volatile boolean doingWork;
+
public SelfStopExecutable() {
super();
}
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ doingWork = true;
try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- if (isDiscarded()) {
- return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
- } else {
+ for (int i = 0; i < 20; i++) {
+ sleepOneSecond();
+
+ if (isDiscarded())
+ return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
+ }
+
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+ } finally {
+ doingWork = false;
+ }
+ }
+
+ private void sleepOneSecond() {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
}
}
+ public void waitForDoWork() {
+ while (doingWork) {
+ sleepOneSecond();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index bcd6a59..2416311 100644
--- a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -97,7 +97,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
@Test
public void testDiscard() throws Exception {
DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SelfStopExecutable();
+ SelfStopExecutable task1 = new SelfStopExecutable();
job.addTask(task1);
jobService.addJob(job);
waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
@@ -105,8 +105,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
waitForJobFinish(job.getId());
Assert.assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
Assert.assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
- Thread.sleep(5000);
- System.out.println(job);
+ task1.waitForDoWork();
}
@SuppressWarnings("rawtypes")
[25/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by bi...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/169c3de6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/169c3de6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/169c3de6
Branch: refs/heads/KYLIN-2361
Commit: 169c3de6456f394d049e9550d65a98f6aa0a316f
Parents: 68e87f0
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 23 13:01:20 2017 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 19 ++--
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 37 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 27 +++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
39 files changed, 366 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -32,11 +32,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,6 +56,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -92,10 +91,10 @@ public class BuildCubeWithEngine {
logger.error("error", e);
exitCode = 1;
}
-
+
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
-
+
System.exit(exitCode);
}
@@ -279,7 +278,7 @@ public class BuildCubeWithEngine {
String cubeName = "ci_inner_join_cube";
clearSegment(cubeName);
-
+
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
@@ -334,10 +333,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e590fd7..439566a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.1.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index ea68855..8095bf8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 1d520c4..d1a1384 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -58,11 +59,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d28c87c..c8c87cb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -27,9 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -395,33 +393,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 98eb7cb..7ce38ea 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -49,11 +49,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -164,14 +164,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -197,14 +196,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -216,12 +214,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 335bfe7..53c95cb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 6217350..1c45967 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,14 +30,15 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -237,13 +238,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,8 +264,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -274,7 +272,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -287,7 +285,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -308,7 +305,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -316,7 +313,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -335,7 +332,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
@@ -360,7 +357,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -368,8 +365,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index df1817e..cad5a3f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -51,10 +52,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +118,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -172,7 +173,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 21a0efb..810747f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index eacff9f..df3cf08 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 68c0a39..581de38 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -26,19 +26,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -89,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -233,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -326,8 +327,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -449,11 +450,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -473,7 +474,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -504,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -539,13 +539,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -562,7 +561,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 8bd4abf..20d0f7d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,9 +130,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -189,10 +188,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
[26/47] kylin git commit: KYLIN-2413 a new profile for cdh binary
packages
Posted by bi...@apache.org.
KYLIN-2413 a new profile for cdh binary packages
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb73da7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb73da7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb73da7a
Branch: refs/heads/KYLIN-2361
Commit: eb73da7a0b27540af54fec23b561b390405620c5
Parents: 169c3de
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jan 23 11:30:13 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 23 13:03:29 2017 +0800
----------------------------------------------------------------------
build/script/build.sh | 2 +-
build/script/package.sh | 11 +-
.../kylin/engine/mr/steps/MockupMapContext.java | 11 +-
pom.xml | 114 +++++++++++++++++++
.../storage/hbase/steps/MockupMapContext.java | 17 +--
tool/pom.xml | 10 ++
6 files changed, 145 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/build.sh
----------------------------------------------------------------------
diff --git a/build/script/build.sh b/build/script/build.sh
index c68ca94..458c612 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
dir=$(dirname ${0})
cd ${dir}/../..
-mvn clean install -DskipTests || { exit 1; }
+mvn clean install -DskipTests $@ || { exit 1; }
#package webapp
echo 'package front-end'
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/package.sh
----------------------------------------------------------------------
diff --git a/build/script/package.sh b/build/script/package.sh
index c850ec3..a58a5ce 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -74,9 +74,8 @@ cat << EOF > build/commit_SHA1
EOF
git rev-parse HEAD >> build/commit_SHA1
-sh build/script/build.sh || { exit 1; }
-sh build/script/download-tomcat.sh || { exit 1; }
-sh build/script/download-spark.sh || { exit 1; }
-sh build/script/prepare.sh || { exit 1; }
-sh build/script/compress.sh || { exit 1; }
-
+sh build/script/build.sh $@ || { exit 1; }
+sh build/script/download-tomcat.sh || { exit 1; }
+sh build/script/download-spark.sh || { exit 1; }
+sh build/script/prepare.sh || { exit 1; }
+sh build/script/compress.sh || { exit 1; }
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..92115c9 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
*
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
private Configuration hconf;
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV));
}
- public MockupMapContext(Configuration hconf, Object[] outKV){
+ public MockupMapContext(Configuration hconf, Object[] outKV) {
this.hconf = hconf;
this.outKV = outKV;
}
@@ -109,7 +109,6 @@ public class MockupMapContext implements MapContext{
throw new NotImplementedException();
}
- @Override
public float getProgress() {
throw new NotImplementedException();
}
@@ -224,7 +223,6 @@ public class MockupMapContext implements MapContext{
throw new NotImplementedException();
}
- @Override
public boolean getTaskCleanupNeeded() {
throw new NotImplementedException();
}
@@ -239,7 +237,6 @@ public class MockupMapContext implements MapContext{
throw new NotImplementedException();
}
- @Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
throw new NotImplementedException();
}
@@ -318,4 +315,8 @@ public class MockupMapContext implements MapContext{
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ public boolean userClassesTakesPrecedence() {
+ throw new NotImplementedException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 439566a..520ac0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -355,6 +355,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop2.version}</version>
</dependency>
@@ -812,6 +817,11 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
+
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+ </repository>
</repositories>
<build>
@@ -1168,6 +1178,110 @@
</build>
</profile>
<profile>
+ <id>cdh-5.7</id>
+ <properties>
+ <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
+ <yarn.version>2.6.0-cdh5.7.0</yarn.version>
+ <hive.version>1.1.0-cdh5.7.0</hive.version>
+ <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
+ <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
+ <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <fork>true</fork>
+ <meminitial>1024m</meminitial>
+ <maxmem>2048m</maxmem>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-jamm</id>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <phase>generate-test-resources</phase>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
+ <destFileName>jamm.jar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <configuration>
+ <append>true</append>
+ <destFile>
+ ${sonar.jacoco.reportPath}
+ </destFile>
+ </configuration>
+ <executions>
+ <execution>
+ <id>pre-test</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ <configuration>
+ <propertyName>surefireArgLine</propertyName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>post-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <configuration>
+ <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
+ <excludes>
+ <exclude>**/IT*.java</exclude>
+ </excludes>
+ <systemProperties>
+ <property>
+ <name>useSandbox</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>buildCubeUsingProvidedData</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>log4j.configuration</name>
+ <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
+ </property>
+ </systemProperties>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<!-- This profile adds/overrides few features of the 'apache-release'
profile in the parent pom. -->
<id>apache-release</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d953d9c..536634d 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -44,10 +44,10 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
/**
* @author yangli9
- *
+ *
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext {
+public class MockupMapContext implements MapContext {
private Configuration hconf;
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext {
return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV));
}
- public MockupMapContext(Configuration hconf, Object[] outKV){
+ public MockupMapContext(Configuration hconf, Object[] outKV) {
this.hconf = hconf;
this.outKV = outKV;
}
@@ -100,16 +100,15 @@ public class MockupMapContext implements MapContext {
}
@Override
- public void setStatus(String msg) {
+ public String getStatus() {
throw new NotImplementedException();
}
@Override
- public String getStatus() {
+ public void setStatus(String msg) {
throw new NotImplementedException();
}
- @Override
public float getProgress() {
throw new NotImplementedException();
}
@@ -224,7 +223,6 @@ public class MockupMapContext implements MapContext {
throw new NotImplementedException();
}
- @Override
public boolean getTaskCleanupNeeded() {
throw new NotImplementedException();
}
@@ -239,7 +237,6 @@ public class MockupMapContext implements MapContext {
throw new NotImplementedException();
}
- @Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
throw new NotImplementedException();
}
@@ -318,4 +315,8 @@ public class MockupMapContext implements MapContext {
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ public boolean userClassesTakesPrecedence() {
+ throw new NotImplementedException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 919a903..91040d4 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -60,6 +60,16 @@
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
<!-- Env & Test -->
<dependency>
[15/47] kylin git commit: KYLIN-2396 remove unused code
Posted by bi...@apache.org.
KYLIN-2396 remove unused code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2229c9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2229c9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2229c9c
Branch: refs/heads/KYLIN-2361
Commit: c2229c9c7f83fe5a28e564c37ada5794a67ee3d8
Parents: 8331d8d
Author: lidongsjtu <li...@apache.org>
Authored: Sat Jan 21 19:52:42 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sat Jan 21 19:52:42 2017 +0800
----------------------------------------------------------------------
.../measure/percentile/PercentileContUdf.java | 37 --------------------
.../kylin/rest/controller/BasicController.java | 22 ++++++++++++
.../rest/controller/DiagnosisController.java | 19 ----------
3 files changed, 22 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
deleted file mode 100644
index 4ef6b75..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.percentile;
-
-public class PercentileContUdf {
- public static double init() {
- return 0;
- }
-
- public static double add(double accumulator, double v, double r) {
- return 0;
- }
-
- public static double merge(double accumulator0, double accumulator1) {
- return 0;
- }
-
- public static double result(long accumulator) {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index f61492d..6b4c9fd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -18,11 +18,20 @@
package org.apache.kylin.rest.controller;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.response.ErrorResponse;
import org.slf4j.Logger;
@@ -74,4 +83,17 @@ public class BasicController {
}
}
+ protected void setDownloadResponse(String downloadFile, final HttpServletResponse response) {
+ File file = new File(downloadFile);
+ try (InputStream fileInputStream = new FileInputStream(file); OutputStream output = response.getOutputStream();) {
+ response.reset();
+ response.setContentType("application/octet-stream");
+ response.setContentLength((int) (file.length()));
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"");
+ IOUtils.copyLarge(fileInputStream, output);
+ output.flush();
+ } catch (IOException e) {
+ throw new InternalErrorException("Failed to download file: " + e.getMessage(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
index d16547e..f5e7c24 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
@@ -19,17 +19,12 @@
package org.apache.kylin.rest.controller;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.io.IOUtils;
import org.apache.kylin.metadata.badquery.BadQueryEntry;
import org.apache.kylin.metadata.badquery.BadQueryHistory;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -103,18 +98,4 @@ public class DiagnosisController extends BasicController {
setDownloadResponse(filePath, response);
}
-
- private void setDownloadResponse(String downloadFile, final HttpServletResponse response) {
- File file = new File(downloadFile);
- try (InputStream fileInputStream = new FileInputStream(file); OutputStream output = response.getOutputStream();) {
- response.reset();
- response.setContentType("application/octet-stream");
- response.setContentLength((int) (file.length()));
- response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"");
- IOUtils.copyLarge(fileInputStream, output);
- output.flush();
- } catch (IOException e) {
- throw new InternalErrorException("Failed to create download for diagnosis. " + e.getMessage(), e);
- }
- }
}
[10/47] kylin git commit: KYLIN-2375 change default ehcache size to
256M
Posted by bi...@apache.org.
KYLIN-2375 change default ehcache size to 256M
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e4ae540
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e4ae540
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e4ae540
Branch: refs/heads/KYLIN-2361
Commit: 1e4ae540bf194c7e67bb622c2eb0cd0ff1a6657c
Parents: 2196975
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 19 17:44:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 19 17:44:19 2017 +0800
----------------------------------------------------------------------
server/src/main/resources/ehcache-test.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e4ae540/server/src/main/resources/ehcache-test.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml
index eb39774..bffe27a 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -12,7 +12,7 @@
limitations under the License. See accompanying LICENSE file.
-->
-<ehcache maxBytesLocalHeap="10M">>
+<ehcache maxBytesLocalHeap="256M">>
<cache name="StorageCache"
eternal="false"
timeToIdleSeconds="86400"
[16/47] kylin git commit: KYLIN-2242 write multiple files in
FactDistinctColumnsReducer with MultipleOutputs
Posted by bi...@apache.org.
KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12
Branch: refs/heads/KYLIN-2361
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Sat Jan 21 23:19:50 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HadoopUtil.java | 16 ++
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../kylin/engine/mr/common/BatchConstants.java | 9 +-
.../engine/mr/steps/CreateDictionaryJob.java | 43 ++--
.../engine/mr/steps/FactDistinctColumnsJob.java | 32 ++-
.../mr/steps/FactDistinctColumnsReducer.java | 240 +++++++------------
.../engine/mr/steps/SaveStatisticsStep.java | 10 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 10 +-
8 files changed, 175 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
}
}
+ public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final String filter) throws IOException {
+ FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(filter);
+ }
+ });
+
+ if (fileStatus.length == 1) {
+ return fileStatus[0].getPath();
+ } else {
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
}
public String getStatisticsPath(String jobId) {
- return getRealizationRootPath(jobId) + "/statistics";
+ return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + BatchConstants.CFG_OUTPUT_STATISTICS;
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
String CFG_STATISTICS_ENABLED = "statistics.enabled";
String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for compatibility issue better not change it
String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
- String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
+ String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+ String CFG_OUTPUT_COLUMN = "column";
+ String CFG_OUTPUT_DICT = "dict";
+ String CFG_OUTPUT_STATISTICS = "statistics";
+ String CFG_OUTPUT_PARTITION = "partition";
+
+
/**
* command line ARGuments
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 95d8cb1..e5d053b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -18,15 +18,20 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.DataInputStream;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.commons.cli.Options;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.HadoopUtil;
@@ -63,21 +68,27 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getWorkingFileSystem();
- if (fs.exists(dictFile) == false)
+ Path colDir = new Path(factColumnsInputPath, col.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+
+ Path dictFile = HadoopUtil.getFilterOnlyPath(fs, colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ if (dictFile == null) {
return null;
-
- FSDataInputStream is = null;
- try {
- is = fs.open(dictFile);
- String dictClassName = is.readUTF();
- Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
- dict.readFields(is);
- logger.info("DictionaryProvider read dict from file: " + dictFile);
- return dict;
- } finally {
- IOUtils.closeQuietly(is);
+ }
+
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(HadoopUtil.getCurrentConfiguration(), SequenceFile.Reader.file(dictFile))) {
+ NullWritable key = NullWritable.get();
+ BytesWritable value = new BytesWritable();
+ reader.next(key, value);
+
+ ByteBuffer buffer = new ByteArray(value.getBytes()).asBuffer();
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(buffer))) {
+ String dictClassName = is.readUTF();
+ Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
+ dict.readFields(is);
+ logger.info("DictionaryProvider read dict from file: " + dictFile);
+ return dict;
+ }
}
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index ce01eb6..aded600 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -23,11 +23,16 @@ import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -82,8 +87,8 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
int uhcReducerCount = cube.getConfig().getUHCReducerCount();
int[] uhcIndex = cubeMgr.getUHCIndex(cube.getDescriptor());
- for(int index : uhcIndex) {
- if(index == 1) {
+ for (int index : uhcIndex) {
+ if (index == 1) {
reducerCount += uhcReducerCount - 1;
}
}
@@ -92,7 +97,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
throw new IllegalArgumentException("The max reducer number for FactDistinctColumnsJob is 255, but now it is " + reducerCount + ", decrease 'kylin.engine.mr.uhc-reducer-count'");
}
-
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled);
@@ -117,6 +121,12 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
attachCubeMetadata(cube, job.getConfiguration());
+ /**
+ * don't compress the reducer output so that {@link CreateDictionaryJob} and {@link UpdateCubeInfoAfterBuildStep}
+ * could read the reducer file directly
+ */
+ job.getConfiguration().set(BatchConstants.CFG_MAPRED_OUTPUT_COMPRESS, "false");
+
return waitForCompletion(job);
} finally {
@@ -138,18 +148,22 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
private void setupReducer(Path output, int numberOfReducers) throws IOException {
job.setReducerClass(FactDistinctColumnsReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Text.class);
job.setPartitionerClass(FactDistinctColumnPartitioner.class);
job.setNumReduceTasks(numberOfReducers);
- // important, reducer writes HDFS directly at the moment
- job.setReduceSpeculativeExecution(false);
-
+ //make each reducer output to respective dir
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_COLUMN, SequenceFileOutputFormat.class, NullWritable.class, Text.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_DICT, SequenceFileOutputFormat.class, NullWritable.class, BytesWritable.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_STATISTICS, SequenceFileOutputFormat.class, LongWritable.class, BytesWritable.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_PARTITION, TextOutputFormat.class, NullWritable.class, LongWritable.class);
+
+
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
+ //prevent to create zero-sized default output
+ LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
+
deletePath(job.getConfiguration(), output);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 711d991..5d2fb72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -18,27 +18,25 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -47,7 +45,7 @@ import org.apache.kylin.dict.IDictionaryBuilder;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CubeStatsWriter;
+import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -63,14 +61,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
private List<TblColRef> columnList;
- private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
protected Map<Long, HLLCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
private int samplingPercentage;
- private List<ByteArray> colValues;
private TblColRef col = null;
private boolean isStatistics = false;
private KylinConfig cubeConfig;
@@ -88,10 +84,14 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
public static final String DICT_FILE_POSTFIX = ".rldict";
public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
+ private MultipleOutputs mos;
+
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
+ mos = new MultipleOutputs(context);
+
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
@@ -109,26 +109,20 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (collectStatistics && (taskId == numberOfTasks - 1)) {
// hll
isStatistics = true;
- statisticsOutput = conf.get(BatchConstants.CFG_STATISTICS_OUTPUT);
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
logger.info("Reducer " + taskId + " handling stats");
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
- isStatistics = false;
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
logger.info("Do not have partition col. This reducer will keep empty");
}
- colValues = Lists.newLinkedList();
- logger.info("Reducer " + taskId + " handling partition column " + col);
} else {
// normal col
- isStatistics = false;
col = columnList.get(reducerIdToColumnIndex.get(taskId));
- colValues = Lists.newLinkedList();
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
@@ -194,15 +188,13 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
logAFewRows(value);
builder.addValue(value);
} else {
- colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
- if (colValues.size() == 1000000) { //spill every 1 million
- logger.info("spill values to disk...");
- outputDistinctValues(col, colValues, context);
- colValues.clear();
- }
+ byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
+ // output written to baseDir/colName/-r-00000 (etc)
+ String fileName = col.getName() + "/";
+ mos.write(BatchConstants.CFG_OUTPUT_COLUMN, NullWritable.get(), new Text(keyBytes), fileName);
}
}
-
+
rowCount++;
}
@@ -212,162 +204,104 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
- private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
- final Configuration conf = context.getConfiguration();
- final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getIdentity());
- final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
- final Path outputFile = new Path(colDir, fileName);
-
- FSDataOutputStream out = null;
- try {
- if (!fs.exists(colDir)) {
- fs.mkdirs(colDir);
- }
-
- if (fs.exists(outputFile)) {
- out = fs.append(outputFile);
- logger.info("append file " + outputFile);
- } else {
- out = fs.create(outputFile);
- logger.info("create file " + outputFile);
- }
-
- for (ByteArray value : values) {
- out.write(value.array(), value.offset(), value.length());
- out.write('\n');
- }
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
- final String fileName = col.getIdentity() + DICT_FILE_POSTFIX;
- FSDataOutputStream out = getOutputStream(context, fileName);
- try {
- String dictClassName = dict.getClass().getName();
- out.writeUTF(dictClassName);
- dict.write(out);
- logger.info("reducer id is:+" + taskId + " colName:" + col.getName() + " writing dict at file : " + fileName + " dict class:" + dictClassName);
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private void outputPartitionInfo(Context context) throws IOException {
- final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX;
- FSDataOutputStream out = getOutputStream(context, fileName);
- try {
- out.writeLong(timeMinValue);
- out.writeLong(timeMaxValue);
- logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
- final Configuration conf = context.getConfiguration();
- final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
- final Path outputFile = new Path(outputPath, outputFileName);
- if (!fs.exists(outputPath)) {
- fs.mkdirs(outputPath);
- }
- FSDataOutputStream out = fs.create(outputFile);
- return out;
- }
-
@Override
protected void doCleanup(Context context) throws IOException, InterruptedException {
if (isStatistics) {
- // output the hll info
- long grandTotal = 0;
- for (HLLCounter hll : cuboidHLLMap.values()) {
- grandTotal += hll.getCountEstimate();
- }
- double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
-
- int mapperNumber = baseCuboidRowCountInMappers.size();
+ //output the hll info;
+ List<Long> allCuboids = Lists.newArrayList();
+ allCuboids.addAll(cuboidHLLMap.keySet());
+ Collections.sort(allCuboids);
- writeMapperAndCuboidStatistics(context); // for human check
- CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
- cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
+ logMapperAndCuboidStatistics(allCuboids); // for human check
+ outputStatistics(allCuboids);
} else if (isPartitionCol) {
// partition col
- if (col != null) {
- outputPartitionInfo(context);
- }
+ outputPartitionInfo();
} else {
// normal col
if (isReducerLocalBuildDict) {
Dictionary<String> dict = builder.build();
- outputDict(col, dict, context);
- } else {
- if (colValues.size() > 0) {
- outputDistinctValues(col, colValues, context);
- colValues.clear();
- }
+ outputDict(col, dict);
}
}
+
+ mos.close();
}
- private void writeMapperAndCuboidStatistics(Context context) throws IOException {
- Configuration conf = context.getConfiguration();
- FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- Path path = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME);
- FSDataOutputStream out = fs.create(path);
+ private void outputPartitionInfo() throws IOException, InterruptedException {
+ if (col != null) {
+ // output written to baseDir/colName/colName.pci-r-00000 (etc)
+ String partitionFileName = col.getName() + "/" + col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
- try {
- String msg;
+ mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMinValue), partitionFileName);
+ mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMaxValue), partitionFileName);
+ logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
+ }
+ }
- List<Long> allCuboids = Lists.newArrayList();
- allCuboids.addAll(cuboidHLLMap.keySet());
- Collections.sort(allCuboids);
+ private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
+ // output written to baseDir/colName/colName.rldict-r-00000 (etc)
+ String dictFileName = col.getName() + "/" + col.getName() + DICT_FILE_POSTFIX;
- msg = "Total cuboid number: \t" + allCuboids.size();
- writeLine(out, msg);
- msg = "Samping percentage: \t" + samplingPercentage;
- writeLine(out, msg);
-
- writeLine(out, "The following statistics are collected based on sampling data.");
- writeLine(out, "Number of Mappers: " + baseCuboidRowCountInMappers.size());
- for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
- if (baseCuboidRowCountInMappers.get(i) > 0) {
- msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
- writeLine(out, msg);
- }
- }
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) {
+ outputStream.writeUTF(dict.getClass().getName());
+ dict.write(outputStream);
- long grantTotal = 0;
- for (long i : allCuboids) {
- grantTotal += cuboidHLLMap.get(i).getCountEstimate();
- msg = "Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate();
- writeLine(out, msg);
- }
+ mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new BytesWritable(baos.toByteArray()), dictFileName);
+ }
+ }
- msg = "Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge;
- writeLine(out, msg);
+ private void outputStatistics(List<Long> allCuboids) throws IOException, InterruptedException {
+ // output written to baseDir/statistics/statistics-r-00000 (etc)
+ String statisticsFileName = BatchConstants.CFG_OUTPUT_STATISTICS + "/" + BatchConstants.CFG_OUTPUT_STATISTICS;
- msg = "After merge, the cube has row count: \t " + grantTotal;
- writeLine(out, msg);
+ ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
- if (grantTotal > 0) {
- msg = "The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal;
- writeLine(out, msg);
- }
+ // mapper overlap ratio at key -1
+ long grandTotal = 0;
+ for (HLLCounter hll : cuboidHLLMap.values()) {
+ grandTotal += hll.getCountEstimate();
+ }
+ double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-1), new BytesWritable(Bytes.toBytes(mapperOverlapRatio)), statisticsFileName);
+
+ // mapper number at key -2
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-2), new BytesWritable(Bytes.toBytes(baseCuboidRowCountInMappers.size())), statisticsFileName);
+
+ // sampling percentage at key 0
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(0L), new BytesWritable(Bytes.toBytes(samplingPercentage)), statisticsFileName);
- } finally {
- IOUtils.closeQuietly(out);
+ for (long i : allCuboids) {
+ valueBuf.clear();
+ cuboidHLLMap.get(i).writeRegisters(valueBuf);
+ valueBuf.flip();
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()), statisticsFileName);
}
}
- private void writeLine(FSDataOutputStream out, String msg) throws IOException {
- out.write(msg.getBytes());
- out.write('\n');
+ private void logMapperAndCuboidStatistics(List<Long> allCuboids) throws IOException {
+ logger.info("Total cuboid number: \t" + allCuboids.size());
+ logger.info("Samping percentage: \t" + samplingPercentage);
+ logger.info("The following statistics are collected based on sampling data.");
+ logger.info("Number of Mappers: " + baseCuboidRowCountInMappers.size());
+ for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
+ if (baseCuboidRowCountInMappers.get(i) > 0) {
+ logger.info("Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i));
+ }
+ }
+
+ long grantTotal = 0;
+ for (long i : allCuboids) {
+ grantTotal += cuboidHLLMap.get(i).getCountEstimate();
+ logger.info("Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate());
+ }
+
+ logger.info("Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge);
+ logger.info("After merge, the cube has row count: \t " + grantTotal);
+ if (grantTotal > 0) {
+ logger.info("The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 2671042..28f99fb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -60,9 +60,11 @@ public class SaveStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
FileSystem fs = HadoopUtil.getWorkingFileSystem();
- Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- if (!fs.exists(statisticsFilePath))
- throw new IOException("File " + statisticsFilePath + " does not exists");
+ Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
+ Path statisticsFilePath = HadoopUtil.getFilterOnlyPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS);
+ if (statisticsFilePath == null) {
+ throw new IOException("fail to find the statistics file in base dir: " + statisticsDir);
+ }
FSDataInputStream is = fs.open(statisticsFilePath);
try {
@@ -110,7 +112,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
+
// in-mem cubing is good when
// 1) the cluster has enough mapper slots to run in parallel
// 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dc80399..81d5c42 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -80,8 +80,13 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
+ Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+ Path outputFile = HadoopUtil.getFilterOnlyPath(fs, colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ if (outputFile == null) {
+ throw new IOException("fail to find the partition file in base dir: " + colDir);
+ }
+
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
@@ -97,5 +102,4 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
segment.setDateRangeStart(minValue);
segment.setDateRangeEnd(maxValue);
}
-
}
[42/47] kylin git commit: KYLIN-2414 minor rename on properties
Posted by bi...@apache.org.
KYLIN-2414 minor rename on properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1
Branch: refs/heads/KYLIN-2361
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 26 10:53:19 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 8 +++++++-
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++--
.../mr/steps/FactDistinctColumnsReducer.java | 19 +++++++++----------
3 files changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
### CUBE | DICTIONARY ###
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
kylin.cube.algorithm=auto
# A smaller threshold prefers layer, a larger threshold prefers in-mem
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
}
- public boolean isReducerLocalBuildDict() {
- return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true"));
+ public boolean isBuildDictInReducerEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", "true"));
}
public String getYarnStatusCheckUrl() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private int rowCount = 0;
//local build dict
- private boolean isReducerLocalBuildDict;
+ private boolean buildDictInReducer;
private IDictionaryBuilder builder;
private long timeMaxValue = Long.MIN_VALUE;
private long timeMinValue = Long.MAX_VALUE;
@@ -119,30 +119,29 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
- logger.info("Do not have partition col. This reducer will keep empty");
+ logger.info("No partition col. This reducer will do nothing");
}
} else {
// normal col
col = columnList.get(reducerIdToColumnIndex.get(taskId));
-
Preconditions.checkNotNull(col);
// local build dict
- isReducerLocalBuildDict = config.isReducerLocalBuildDict();
+ buildDictInReducer = config.isBuildDictInReducerEnabled();
if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
- isReducerLocalBuildDict = false;
+ buildDictInReducer = false;
}
if(config.getUHCReducerCount() > 1) {
int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
int colIndex = reducerIdToColumnIndex.get(taskId);
if (uhcIndex[colIndex] == 1)
- isReducerLocalBuildDict = false; //for UHC columns, this feature should be disabled
+ buildDictInReducer = false; //for UHC columns, this feature should be disabled
}
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
}
- logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+ logger.info("Reducer " + taskId + " handling column " + col + ", buildDictInReducer=" + buildDictInReducer);
}
}
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
timeMaxValue = Math.max(timeMaxValue, time);
} else {
// normal col
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
logAFewRows(value);
builder.addValue(value);
@@ -228,7 +227,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
outputPartitionInfo();
} else {
// normal col
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
Dictionary<String> dict = builder.build();
outputDict(col, dict);
}
[40/47] kylin git commit: KYLIN-2374 code review
Posted by bi...@apache.org.
KYLIN-2374 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eae37ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eae37ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eae37ef
Branch: refs/heads/KYLIN-2361
Commit: 5eae37ef18ca51027c6bb2cfd3410fefc7982f2a
Parents: a2a59c4
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 26 09:55:48 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 26 09:55:48 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 3 +-
build/deploy/spark-defaults.conf | 1 -
.../apache/kylin/common/KylinConfigBase.java | 8 --
.../kylin/common/persistence/ResourceStore.java | 3 +
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../ExtendedColumnMeasureType.java | 8 +-
.../storage/hdfs/ITHDFSResourceStoreTest.java | 36 +++++++-
.../kylin/storage/hbase/HBaseResourceStore.java | 3 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 90 +++++++++++---------
9 files changed, 97 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index eceb886..43ea17d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -211,8 +211,9 @@ kylin.engine.spark-conf.spark.executor.memory=4G
kylin.engine.spark-conf.spark.executor.cores=4
kylin.engine.spark-conf.spark.executor.instances=8
kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
-kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.enabled=true
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/deploy/spark-defaults.conf
----------------------------------------------------------------------
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
index 36c0ab3..78a4bc9 100644
--- a/build/deploy/spark-defaults.conf
+++ b/build/deploy/spark-defaults.conf
@@ -1,5 +1,4 @@
spark.yarn.submit.file.replication=1
-spark.eventLog.enabled=true
spark.yarn.max.executor.failures=3
spark.driver.extraJavaOptions=-Dhdp.version=current
spark.yarn.am.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5932197..b1acbbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,14 +193,6 @@ abstract public class KylinConfigBase implements Serializable {
return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
- public String getRawHdfsWorkingDirectory() {
- String root = getRequired("kylin.env.hdfs-working-dir");
- if (!root.endsWith("/")) {
- root += "/";
- }
- return root;
- }
-
// ============================================================================
// METADATA
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 25a0801..c441618 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -63,6 +63,9 @@ abstract public class ResourceStore {
public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
public static final String BAD_QUERY_RESOURCE_ROOT = "/bad_query";
+
+ protected static final String DEFAULT_STORE_NAME = "kylin_metadata";
+
private static final ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
private static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 7e599da..5e970bf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -891,7 +891,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
func.init(model);
allColumns.addAll(func.getParameter().getColRefs());
- if (ExtendedColumnMeasureType.FUNC_RAW.equalsIgnoreCase(m.getFunction().getExpression())) {
+ if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) {
FunctionDesc functionDesc = m.getFunction();
List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index 1b2cda3..de5ee25 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -47,8 +47,8 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
private static final Logger logger = LoggerFactory.getLogger(ExtendedColumnMeasureType.class);
- public static final String FUNC_RAW = "EXTENDED_COLUMN";
- public static final String DATATYPE_RAW = "extendedcolumn";
+ public static final String FUNC_EXTENDED_COLUMN = "EXTENDED_COLUMN";
+ public static final String DATATYPE_EXTENDED_COLUMN = "extendedcolumn";
private final DataType dataType;
public static class Factory extends MeasureTypeFactory<ByteArray> {
@@ -60,12 +60,12 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
@Override
public String getAggrFunctionName() {
- return FUNC_RAW;
+ return FUNC_EXTENDED_COLUMN;
}
@Override
public String getAggrDataTypeName() {
- return DATATYPE_RAW;
+ return DATATYPE_EXTENDED_COLUMN;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ff66048..ec12722 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -18,21 +18,28 @@
package org.apache.kylin.storage.hdfs;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static junit.framework.TestCase.assertTrue;
+
public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
KylinConfig kylinConfig;
+ FileSystem fs;
@Before
public void setup() throws Exception {
this.createTestMetadata();
kylinConfig = KylinConfig.getInstanceFromEnv();
+ fs = HadoopUtil.getWorkingFileSystem();
}
@After
@@ -41,12 +48,37 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
}
@Test
- public void testResourceStoreBasic() throws Exception {
+ public void testBasic() throws Exception {
+ String oldUrl = kylinConfig.getMetadataUrl();
+ String path = "/kylin/kylin_metadata/metadata";
+ kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
+ HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
+ ResourceStoreTest.testAStore(store);
+ kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ assertTrue(fs.exists(new Path(path)));
+ }
+
+ @Test
+ public void testQalifiedName() throws Exception {
String oldUrl = kylinConfig.getMetadataUrl();
- kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hdfs");
+ String path = "hdfs:///kylin/kylin_metadata/metadata_test1";
+ kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
ResourceStoreTest.testAStore(store);
kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ assertTrue(fs.exists(new Path(path)));
}
+ @Test
+ public void testFullQalifiedName() throws Exception {
+ String oldUrl = kylinConfig.getMetadataUrl();
+ String path = "hdfs://sandbox.hortonworks.com:8020/kylin/kylin_metadata/metadata_test2";
+ kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
+ HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
+ ResourceStoreTest.testAStore(store);
+ kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ assertTrue(fs.exists(new Path(path)));
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 0901b54..501f1e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -59,7 +59,6 @@ public class HBaseResourceStore extends ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(HBaseResourceStore.class);
- private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
private static final String FAMILY = "f";
private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
private static final String COLUMN = "c";
@@ -80,7 +79,7 @@ public class HBaseResourceStore extends ResourceStore {
String metadataUrl = kylinConfig.getMetadataUrl();
// split TABLE@HBASE_URL
int cut = metadataUrl.indexOf('@');
- tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+ tableNameBase = cut < 0 ? DEFAULT_STORE_NAME : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
if (!hbaseUrl.equals("hbase"))
throw new IOException("Can not create HBaseResourceStore. Url not match. Url:" + hbaseUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 38acfb0..d24d3b4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -46,11 +46,7 @@ public class HDFSResourceStore extends ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
- private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 10;
-
- private static final String DEFAULT_FOLDER_NAME = "kylin_metadata";
-
- private static final String DEFAULT_METADATA_FOLDER_NAME = "hdfs_metadata";
+ private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 2;
private Path hdfsMetaPath;
@@ -62,42 +58,43 @@ public class HDFSResourceStore extends ResourceStore {
super(kylinConfig);
String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
- String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut);
- String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
- if (!hdfsUrl.equals("hdfs"))
- throw new IOException("Can not create HDFSResourceStore. Url not match. Url:" + hdfsUrl);
- metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
- logger.info("meta dir name :" + metaDirName);
- createMetaFolder(metaDirName, kylinConfig);
- }
-
- private void createMetaFolder(String metaDirName, KylinConfig kylinConfig) throws Exception {
- String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
- fs = HadoopUtil.getFileSystem(hdfsWorkingDir);
- logger.info("hdfs working dir : " + hdfsWorkingDir);
- Path hdfsWorkingPath = new Path(hdfsWorkingDir);
- if (!fs.exists(hdfsWorkingPath)) {
- throw new IOException("HDFS working dir not exist");
+ if (cut < 0) {
+ throw new IOException("kylin.metadata.url not recognized for HDFSResourceStore: " + metadataUrl);
}
+ String suffix = metadataUrl.substring(cut + 1);
+ if (!suffix.equals("hdfs"))
+ throw new IOException("kylin.metadata.url not recognized for HDFSResourceStore:" + suffix);
+
+ String path = metadataUrl.substring(0, cut);
+ fs = HadoopUtil.getFileSystem(path);
+ Path metadataPath = new Path(path);
//creat lock manager
- this.lockManager = new LockManager(kylinConfig, kylinConfig.getRawHdfsWorkingDirectory() + metaDirName);
+ this.lockManager = new LockManager(kylinConfig, getRelativePath(metadataPath));
+ if (fs.exists(metadataPath) == false) {
+ logger.warn("Path not exist in HDFS, create it: " + path);
+ createMetaFolder(metadataPath, kylinConfig);
+ }
+
+ hdfsMetaPath = metadataPath;
+ logger.info("hdfs meta path : " + hdfsMetaPath.toString());
+
+ }
+
+
+
+ private void createMetaFolder(Path metaDirName, KylinConfig kylinConfig) throws Exception {
//create hdfs meta path
- hdfsMetaPath = new Path(hdfsWorkingPath, metaDirName);
- if (!fs.exists(hdfsMetaPath)) {
- ResourceLock lock = lockManager.getLock(lockManager.getLockPath("/"));
- try {
- if (lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES)) {
- logger.info("get root lock successfully");
- if (!fs.exists(hdfsMetaPath)) {
- fs.mkdirs(hdfsMetaPath);
- logger.info("create hdfs meta path");
- }
+ ResourceLock lock = lockManager.getLock(getRelativePath(metaDirName));
+ try {
+ if (lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS)) {
+ if (!fs.exists(metaDirName)) {
+ fs.mkdirs(metaDirName);
}
- } finally {
- lockManager.releaseLock(lock);
}
+ } finally {
+ lockManager.releaseLock(lock);
}
- logger.info("hdfs meta path : " + hdfsMetaPath.toString());
+ logger.info("hdfs meta path created: " + metaDirName.toString());
}
@Override
@@ -170,7 +167,7 @@ public class HDFSResourceStore extends ResourceStore {
ResourceLock lock = null;
try {
lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
in = fs.open(p);
long t = in.readLong();
return t;
@@ -192,7 +189,7 @@ public class HDFSResourceStore extends ResourceStore {
ResourceLock lock = null;
try {
lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
out = fs.create(p, true);
out.writeLong(ts);
IOUtils.copy(content, out);
@@ -228,7 +225,7 @@ public class HDFSResourceStore extends ResourceStore {
ResourceLock lock = null;
try {
lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
Path p = getRealHDFSPath(resPath);
if (fs.exists(p)) {
fs.delete(p, true);
@@ -253,4 +250,21 @@ public class HDFSResourceStore extends ResourceStore {
return new Path(this.hdfsMetaPath, resourcePath);
}
+ private static String getRelativePath(Path hdfsPath) {
+ String path = hdfsPath.toString();
+ int index = path.indexOf("://");
+ if (index > 0) {
+ path = path.substring(index + 3);
+ }
+
+ if (path.startsWith("/") == false) {
+ if (path.indexOf("/") > 0) {
+ path = path.substring(path.indexOf("/"));
+ } else {
+ path = "/" + path;
+ }
+ }
+ return path;
+ }
+
}
[36/47] kylin git commit: minor,
upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169
Posted by bi...@apache.org.
minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/571b7914
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/571b7914
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/571b7914
Branch: refs/heads/KYLIN-2361
Commit: 571b7914f763e05160fb7598bbc2bfbe54d8c7cf
Parents: b6d247c
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jan 24 15:29:05 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Jan 24 15:29:10 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java | 4 ++--
.../java/org/apache/kylin/provision/BuildCubeWithEngine.java | 4 ++--
.../java/org/apache/kylin/provision/BuildCubeWithStream.java | 2 +-
server/src/main/java/org/apache/kylin/rest/DebugTomcat.java | 2 +-
.../apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java | 2 +-
5 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 03509cb..dac6aa9 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -46,7 +46,7 @@ public class DeployLocalMetaToRemoteTest {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
}
}
@@ -70,4 +70,4 @@ public class DeployLocalMetaToRemoteTest {
System.out.println("blank");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cbec4f6..08cc6b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -112,7 +112,7 @@ public class BuildCubeWithEngine {
System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
}
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
@@ -363,4 +363,4 @@ public class BuildCubeWithEngine {
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 194ea35..37ff8ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -274,7 +274,7 @@ public class BuildCubeWithStream {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
}
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 59b7a28..3461e1d 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -48,7 +48,7 @@ public class DebugTomcat {
System.setProperty("catalina.home", ".");
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
- System.err.println("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ System.err.println("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
System.exit(1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index 2356861..691886b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -44,7 +44,7 @@ public class SandboxMetastoreCLI {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
}
if (args.length < 1) {
[32/47] kylin git commit: KYLIN-2374 Allow kylin to store metadata in
HDFS instead of HBase
Posted by bi...@apache.org.
KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d23bf930
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d23bf930
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d23bf930
Branch: refs/heads/KYLIN-2361
Commit: d23bf930da0b542d0e6981917e6bde055839577a
Parents: db85d66
Author: xiefan46 <95...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 5 +
.../common/persistence/ResourceStoreTest.java | 3 +-
.../test_case_data/sandbox/kylin.properties | 1 +
.../storage/hdfs/ITHDFSResourceStoreTest.java | 117 +++++++++++
.../org/apache/kylin/storage/hdfs/HDFSLock.java | 41 ++++
.../kylin/storage/hdfs/HDFSLockManager.java | 45 +++++
.../kylin/storage/hdfs/HDFSResourceStore.java | 198 +++++++++++++++++++
7 files changed, 409 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 05df177..44d636d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.metadata.url");
}
+ //for hdfs resource store
+ public String getHDFSMetadataUrl() {
+ return getOptional("kylin.metadata.hdfs.url", "kylin_default_instance_hdfs@hdfs");
+ }
+
// for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
}
// list
- NavigableSet<String> list;
+ NavigableSet<String> list = null;
list = store.listResources(dir1);
+ System.out.println(list);
assertTrue(list.contains(path1));
assertTrue(list.contains(path2) == false);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 6c512dc..b01c377 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
# The metadata store in hbase
kylin.metadata.url=kylin_default_instance@hbase
+
# The storage for final cube file in hbase
kylin.storage.url=hbase
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 0000000..ef04957
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hdfs;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ResourceStoreTest;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by xiefan on 17-1-10.
+ */
+public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
+
+ KylinConfig kylinConfig;
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Ignore
+ @Test
+ public void testHDFSUrl() throws Exception {
+ assertEquals("kylin_default_instance_hdfs@hdfs", kylinConfig.getHDFSMetadataUrl());
+ System.out.println("hdfs working dir : " + kylinConfig.getHdfsWorkingDirectory());
+ }
+
+
+ @Ignore
+ @Test
+ public void testMultiThreadWriteHDFS() throws Exception{
+ //System.out.println(kylinConfig.getHdfsWorkingDirectory());
+ final Path testDir = new Path("hdfs:///test123");
+ final FileSystem fs = HadoopUtil.getFileSystem(testDir);
+ final String fileName = "test.json";
+ int threadNum = 3;
+ ExecutorService service = Executors.newFixedThreadPool(threadNum);
+ final CountDownLatch latch = new CountDownLatch(threadNum);
+ Path p = new Path(testDir,fileName);
+ fs.deleteOnExit(p);
+ fs.createNewFile(p);
+ for(int i=0;i<threadNum;i++) {
+ service.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ long id = Thread.currentThread().getId();
+ Path p = new Path(testDir, fileName);
+ /*while(fs.exists(p)){
+ System.out.println("Thread id : " + id + " can not get lock,sleep a while");
+ Thread.currentThread().sleep(1000);
+ }*/
+ while(!fs.createNewFile(p)){
+ System.out.println("Thread id : " + id + " can not get lock,sleep a while");
+ Thread.currentThread().sleep(1000);
+ }
+ System.out.println("Thread id : " + id + " get lock, sleep a while");
+ Thread.currentThread().sleep(1000);
+ fs.delete(p,true);
+ System.out.println("Thread id : " + id + " release lock");
+ latch.countDown();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+ Thread.currentThread().sleep(1000);
+ fs.delete(p,true);
+ System.out.println("main thread release lock.Waiting threads down");
+ System.out.println("file still exist : " + fs.exists(p));
+ latch.await();
+ }
+
+ @Test
+ public void testHDFSStore() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ ResourceStore store = new HDFSResourceStore(config);
+ ResourceStoreTest.testAStore(store);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
new file mode 100644
index 0000000..8710edf
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
@@ -0,0 +1,41 @@
+package org.apache.kylin.storage.hdfs;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Created by xiefan on 17-1-17.
+ */
+public class HDFSLock {
+
+ private Path rawLock;
+
+ private static final Logger logger = LoggerFactory.getLogger(HDFSLock.class);
+
+ protected HDFSLock(String resourceFullPath) {
+ this.rawLock = new Path(resourceFullPath);
+ }
+
+ public boolean init(FileSystem fs) throws IOException, InterruptedException {
+ if (!fs.isFile(rawLock)) {
+ logger.info("Not support directory lock yet");
+ return false;
+ }
+ while (!fs.createNewFile(rawLock)) {
+ Thread.currentThread().sleep(1000);
+ }
+ return true;
+ }
+
+ public boolean release(FileSystem fs) throws IOException, InterruptedException {
+ while (!fs.delete(rawLock, false)) {
+ Thread.currentThread().sleep(1000);
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
new file mode 100644
index 0000000..1cd0800
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.storage.hdfs;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.engine.mr.HadoopUtil;
+
+import java.io.IOException;
+
+/**
+ * Created by xiefan on 17-1-17.
+ */
+public class HDFSLockManager {
+
+ private static final String LOCK_HOME = "LOCK_HOME";
+
+ private Path lockPath;
+
+ private FileSystem fs;
+
+ public HDFSLockManager(String hdfsWorkingDir) throws IOException{
+ this.lockPath = new Path(hdfsWorkingDir,LOCK_HOME);
+ this.fs = HadoopUtil.getFileSystem(lockPath);
+ if(!fs.exists(lockPath)){
+ fs.create(lockPath);
+ }
+ }
+
+ public HDFSLock getLock(String resourceFullPath) throws IOException,InterruptedException,IllegalStateException{
+ HDFSLock lock = new HDFSLock(resourceFullPath);
+ boolean success = lock.init(fs);
+ if(success){
+ return lock;
+ }else{
+ throw new IllegalStateException("Try get lock fail. Resourse path : " + resourceFullPath);
+ }
+ }
+
+ public void releaseLock(HDFSLock lock) throws IOException,InterruptedException,IllegalStateException{
+ boolean success = lock.release(fs);
+ if(!success)
+ throw new IllegalStateException("Release lock fail");
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
new file mode 100644
index 0000000..c7f0f25
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hdfs;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.RawResource;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+/**
+ * Created by xiefan on 17-1-10.
+ */
+public class HDFSResourceStore extends ResourceStore {
+
+ private static final String DEFAULT_TABLE_NAME = "kylin_default_instance_hdfs";
+
+ private Path hdfsMetaPath;
+
+ private FileSystem fs;
+
+ private HDFSLockManager lockManager;
+
+ private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
+
+ //public for test. Normal should be protected
+ public HDFSResourceStore(KylinConfig kylinConfig) throws IOException {
+ super(kylinConfig);
+ String metadataUrl = kylinConfig.getHDFSMetadataUrl();
+ // split TABLE@HBASE_URL
+ int cut = metadataUrl.indexOf('@');
+ String metaDirName = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+ createMetaFolder(metaDirName, kylinConfig);
+ }
+
+ private void createMetaFolder(String metaDirName, KylinConfig kylinConfig) throws IOException {
+ String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
+ fs = HadoopUtil.getFileSystem(hdfsWorkingDir);
+ Path hdfsWorkingPath = new Path(hdfsWorkingDir);
+ if (!fs.exists(hdfsWorkingPath)) {
+ throw new IOException("HDFS working dir not exist");
+ }
+ hdfsMetaPath = new Path(hdfsWorkingPath, metaDirName);
+ if (!fs.exists(hdfsMetaPath)) {
+ fs.create(hdfsMetaPath, true);
+ }
+ lockManager = new HDFSLockManager(hdfsWorkingDir);
+ }
+
+ @Override
+ protected NavigableSet<String> listResourcesImpl(String folderPath) throws IOException {
+ Path p = getRealHDFSPath(folderPath);
+ if (!fs.exists(p) || !fs.isDirectory(p)) {
+ return null;
+ }
+ TreeSet<String> r = new TreeSet<>();
+ FileStatus[] statuses = fs.listStatus(p);
+ String prefix = folderPath.endsWith("/") ? folderPath : folderPath + "/";
+ for (FileStatus status : statuses) {
+ r.add(prefix + status.getPath().getName());
+ }
+ return r;
+ }
+
+ @Override
+ protected boolean existsImpl(String resPath) throws IOException {
+ Path p = getRealHDFSPath(resPath);
+ return fs.exists(p) && fs.isFile(p);
+ }
+
+ @Override
+ protected List<RawResource> getAllResourcesImpl(String folderPath, long timeStart, long timeEndExclusive) throws IOException {
+ NavigableSet<String> resources = listResources(folderPath);
+ if (resources == null)
+ return Collections.emptyList();
+ List<RawResource> result = Lists.newArrayListWithCapacity(resources.size());
+ try {
+ for (String res : resources) {
+ long ts = getResourceTimestampImpl(res);
+ if (timeStart <= ts && ts < timeEndExclusive) {
+ RawResource resource = getResourceImpl(res);
+ if (resource != null) // can be null if is a sub-folder
+ result.add(resource);
+ }
+ }
+ } catch (IOException ex) {
+ for (RawResource rawResource : result) {
+ IOUtils.closeQuietly(rawResource.inputStream);
+ }
+ throw ex;
+ }
+ return result;
+ }
+
+ @Override
+ protected RawResource getResourceImpl(String resPath) throws IOException {
+ Path p = getRealHDFSPath(resPath);
+ if (fs.exists(p) && fs.isFile(p)) {
+ if (fs.getFileStatus(p).getLen() == 0) {
+ logger.warn("Zero length file: " + p.toString());
+ }
+ FSDataInputStream in = fs.open(p);
+ return new RawResource(fs.open(p), getResourceTimestamp(resPath));
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ protected long getResourceTimestampImpl(String resPath) throws IOException {
+ Path p = getRealHDFSPath(resPath);
+ if (!fs.exists(p) || !fs.isFile(p)) {
+ return 0;
+ }
+ FileStatus status = fs.getFileStatus(p);
+ return status.getModificationTime();
+ }
+
+ @Override
+ protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+ Path p = getRealHDFSPath(resPath);
+ FSDataOutputStream out = null;
+ try {
+ out = fs.create(p, true);
+ IOUtils.copy(content, out);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ @Override
+ protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
+ Path p = getRealHDFSPath(resPath);
+ if (!fs.exists(p)) {
+ if (oldTS != 0) {
+ throw new IllegalStateException("For not exist file. OldTS have to be 0. but Actual oldTS is : " + oldTS);
+ }
+
+ } else {
+ long realLastModify = getResourceTimestamp(resPath);
+ if (realLastModify != oldTS) {
+ throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but found " + realLastModify);
+ }
+ }
+ putResourceImpl(resPath, new ByteArrayInputStream(content), newTS);
+ return newTS;
+ }
+
+ @Override
+ protected void deleteResourceImpl(String resPath) throws IOException {
+ Path p = getRealHDFSPath(resPath);
+ if (fs.exists(p)) {
+ fs.delete(p, true);
+ }
+ }
+
+ @Override
+ protected String getReadableResourcePathImpl(String resPath) {
+ return getRealHDFSPath(resPath).toString();
+ }
+
+ private Path getRealHDFSPath(String resourcePath) {
+ return new Path(this.hdfsMetaPath, resourcePath);
+ }
+
+}
[04/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and
Avatica to 1.9.0
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
</goals>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
+
<artifactSet>
- <!-- jackson is already packaged into calcite-avatica.jar. To avoid including jackson
- twice, we include calcite-avatica which has jackson and exclude jackson. -->
<excludes>
- <exclude>com.fasterxml.jackson.core:*</exclude>
+ <exclude>com.google.protobuf:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ <exclude>commons-codec:*</exclude>
</excludes>
</artifactSet>
+
<relocations>
<relocation>
<pattern>org.apache.calcite</pattern>
@@ -86,10 +88,6 @@
<pattern>org.apache.http</pattern>
<shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
</relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern>
- </relocation>
</relocations>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
- columns.add(columnMetaData(name, index, field.getType()));
+ columns.add(columnMetaData(name, index, field.getType(), true));
fields.add(field);
fieldNames.add(fieldName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>atopcalcite</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>avatica-core</artifactId>
+ <groupId>org.apache.calcite.avatica</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -74,14 +80,6 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-query</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-linq4j</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
<!-- Env & Test -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
import java.io.File;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
+import java.sql.Driver;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
<aspectj.version>1.8.9</aspectj.version>
<!-- Calcite Version -->
- <calcite.version>1.10.0</calcite.version>
- <avatica.version>1.8.0</avatica.version>
+ <calcite.version>1.11.0</calcite.version>
+ <avatica.version>1.9.0</avatica.version>
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
@@ -450,15 +450,9 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-linq4j</artifactId>
- <version>${calcite.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
<version>${avatica.version}</version>
-
</dependency>
<!-- Workaround for hive 0.14 avatica dependency -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>atopcalcite</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
[22/47] kylin git commit: KYLIN 2396 GUI for Percentile pre
aggregation implementation
Posted by bi...@apache.org.
KYLIN 2396 GUI for Percentile pre aggregation implementation
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28673a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28673a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28673a81
Branch: refs/heads/KYLIN-2361
Commit: 28673a8126ec2ecff8c125c0ecf887ce2565362c
Parents: 6aaf667
Author: chenzhx <34...@qq.com>
Authored: Fri Jan 20 18:00:20 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 23 11:32:46 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 18 +++++++
webapp/app/js/model/cubeConfig.js | 2 +-
webapp/app/partials/cubeDesigner/measures.html | 54 ++++++++-------------
3 files changed, 40 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index c8548ce..7c155fe 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -80,6 +80,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.updateMeasureStatus.editIndex = index;
}
$scope.nextParameters = [];
+ $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
$scope.newMeasure = (!!measure)? jQuery.extend(true, {},measure):CubeDescModel.createMeasure();
if(!!measure && measure.function.parameter.next_parameter){
$scope.nextPara.value = measure.function.parameter.next_parameter.value;
@@ -93,6 +94,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
}else{
$scope.newMeasure.showDim=false;
}
+ $scope.measureParamValueUpdate();
if($scope.newMeasure.function.expression=="TOP_N"){
$scope.convertedColumns=[];
if($scope.newMeasure.function.configuration==null){
@@ -326,6 +328,19 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
return false;
}
}
+
+ $scope.measureParamValueUpdate = function(){
+ if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && $scope.newMeasure.showDim==true){
+ $scope.measureParamValueColumn=$scope.getAllModelDimMeasureColumns();
+ }
+ if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && $scope.newMeasure.showDim==false){
+ $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
+ }
+ if($scope.newMeasure.function.expression == 'EXTENDED_COLUMN'){
+ $scope.measureParamValueColumn=$scope.getExtendedHostColumn();
+ }
+ }
+
//map right return type for param
$scope.measureReturnTypeUpdate = function(){
@@ -393,6 +408,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
case "COUNT":
$scope.newMeasure.function.returntype = "bigint";
break;
+ case "PERCENTILE":
+ $scope.newMeasure.function.returntype = "percentile(100)";
+ break;
default:
$scope.newMeasure.function.returntype = "";
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 87413dc..35af1fc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
//~ Define metadata & class
measureParamType: ['column', 'constant'],
- measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 'RAW','EXTENDED_COLUMN'],
+ measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 'RAW','EXTENDED_COLUMN','PERCENTILE'],
dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
cubePartitionTypes: ['APPEND'],
joinTypes: [
http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 4d2d8ab..4a48843 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -104,7 +104,7 @@
<div class="col-xs-12 col-sm-6">
<select class="form-control"
ng-init="newMeasure.function.expression = (!!newMeasure.function.expression)?newMeasure.function.expression:cubeConfig.dftSelections.measureExpression" chosen ng-model="newMeasure.function.expression" required
- ng-change="measureReturnTypeUpdate();"
+ ng-change="measureReturnTypeUpdate();measureParamValueUpdate();"
ng-options="me as me for me in cubeConfig.measureExpressions">
<option value=""></option>
</select>
@@ -144,35 +144,23 @@
<div class="col-xs-12 col-sm-6">
- <span class="font-color-default"
- ng-if="newMeasure.function.parameter.type == 'constant'"
- ng-init="newMeasure.function.parameter.value = 1"><b> 1</b></span>
- <!--!COUNT_DISTINCT-->
-
- <select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression !== 'EXTENDED_COLUMN' && newMeasure.showDim==true "
- ng-model="newMeasure.function.parameter.value" required
- ng-change="measureReturnTypeUpdate();"
- ng-options="column as column for column in getAllModelDimMeasureColumns(newMeasure)" >
- <option value="">-- Select a Column --</option>
- </select>
-
+ <span class="font-color-default"
+ ng-if="newMeasure.function.parameter.type == 'constant'"
+ ng-init="newMeasure.function.parameter.value = 1">
+ <b> 1</b>
+ </span>
+ <!--!COUNT_DISTINCT-->
<select class="form-control" chosen
- ng-if="newMeasure.function.parameter.type == 'column' && newMeasure.function.expression !== 'EXTENDED_COLUMN' && newMeasure.showDim==false "
- ng-model="newMeasure.function.parameter.value" required
- ng-change="measureReturnTypeUpdate();"
- ng-options="column as column for column in getCommonMetricColumns(newMeasure)" >
- <option value="">-- Select a Column --</option>
- </select>
- <select class="form-control" chosen
- ng-if="newMeasure.function.expression == 'EXTENDED_COLUMN'"
- ng-model="newMeasure.function.parameter.value" required
- ng-change="measureReturnTypeUpdate();"
- ng-options="column as column for column in getExtendedHostColumn()" >
- <option value="">-- Select a Column --</option>
- </select>
- </div>
- <label ng-if="newMeasure.function.parameter.type == 'column'&& newMeasure.function.expression !== 'EXTENDED_COLUMN'"><input type="checkbox" ng-model="newMeasure.showDim" /> Also Show Dimensions</label>
+ ng-if="newMeasure.function.parameter.type == 'column'"
+ ng-model="newMeasure.function.parameter.value" required
+ ng-change="measureReturnTypeUpdate();"
+ ng-options="column as column for column in measureParamValueColumn" >
+ <option value="">{{dataKylin.cube.cubeMSParamValueSelect}}</option>
+ </select>
+ </div>
+ <label ng-if="newMeasure.function.parameter.type == 'column'&& newMeasure.function.expression !== 'EXTENDED_COLUMN'">
+ <input type="checkbox" ng-model="newMeasure.showDim" ng-change="measureParamValueUpdate();" /> Also Show Dimensions
+ </label>
</div>
</div>
@@ -221,10 +209,10 @@
ng-init="newMeasure.function.returntype=newMeasure.function.returntype?newMeasure.function.returntype:'extendedcolumn(100)'"
ng-model="newMeasure.function.returntype" required />
- <span class="font-color-default"
- ng-if="newMeasure.function.expression != 'COUNT_DISTINCT' && newMeasure.function.expression != 'TOP_N' && newMeasure.function.expression != 'EXTENDED_COLUMN' "
- ><b> {{newMeasure.function.returntype | uppercase}}</b>
- </span>
+ <span class="font-color-default"
+ ng-if="newMeasure.function.expression != 'COUNT_DISTINCT' && newMeasure.function.expression != 'TOP_N' && newMeasure.function.expression != 'EXTENDED_COLUMN' ">
+ <b> {{newMeasure.function.returntype | uppercase}}</b>
+ </span>
</div>
</div>
</div>
[24/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
- private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -386,7 +387,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 61c73d5..1cdb2f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -235,9 +236,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -257,7 +258,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index 1db60fb..8dd2164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -120,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -136,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -172,8 +172,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -208,7 +208,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -226,8 +226,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 1ea8e8d..bba6745 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -59,12 +60,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 62af2c9..0784305 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -57,6 +59,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,7 +80,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -153,22 +157,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c8bff89..c0042f3 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -231,6 +231,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -447,11 +448,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -471,7 +472,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -537,13 +537,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -560,7 +559,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 19e5db0..f52fc3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[17/47] kylin git commit: Revert "KYLIN-2242 write multiple files in
FactDistinctColumnsReducer with MultipleOutputs"
Posted by bi...@apache.org.
Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"
This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f
Branch: refs/heads/KYLIN-2361
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi <sh...@apache.org>
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Jan 22 11:38:17 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HadoopUtil.java | 16 --
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../kylin/engine/mr/common/BatchConstants.java | 9 +-
.../engine/mr/steps/CreateDictionaryJob.java | 43 ++--
.../engine/mr/steps/FactDistinctColumnsJob.java | 32 +--
.../mr/steps/FactDistinctColumnsReducer.java | 240 ++++++++++++-------
.../engine/mr/steps/SaveStatisticsStep.java | 10 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 10 +-
8 files changed, 187 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
}
}
- public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final String filter) throws IOException {
- FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith(filter);
- }
- });
-
- if (fileStatus.length == 1) {
- return fileStatus[0].getPath();
- } else {
- return null;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
}
public String getStatisticsPath(String jobId) {
- return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + BatchConstants.CFG_OUTPUT_STATISTICS;
+ return getRealizationRootPath(jobId) + "/statistics";
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
String CFG_STATISTICS_ENABLED = "statistics.enabled";
String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for compatibility issue better not change it
String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+ String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
- String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
- String CFG_OUTPUT_COLUMN = "column";
- String CFG_OUTPUT_DICT = "dict";
- String CFG_OUTPUT_STATISTICS = "statistics";
- String CFG_OUTPUT_PARTITION = "partition";
-
-
/**
* command line ARGuments
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index e5d053b..95d8cb1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -18,20 +18,15 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.DataInputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.commons.cli.Options;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ByteBufferBackedInputStream;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.HadoopUtil;
@@ -68,27 +63,21 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path colDir = new Path(factColumnsInputPath, col.getName());
- FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
-
- Path dictFile = HadoopUtil.getFilterOnlyPath(fs, colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- if (dictFile == null) {
+ Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
+ if (fs.exists(dictFile) == false)
return null;
- }
-
- try (SequenceFile.Reader reader = new SequenceFile.Reader(HadoopUtil.getCurrentConfiguration(), SequenceFile.Reader.file(dictFile))) {
- NullWritable key = NullWritable.get();
- BytesWritable value = new BytesWritable();
- reader.next(key, value);
-
- ByteBuffer buffer = new ByteArray(value.getBytes()).asBuffer();
- try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(buffer))) {
- String dictClassName = is.readUTF();
- Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
- dict.readFields(is);
- logger.info("DictionaryProvider read dict from file: " + dictFile);
- return dict;
- }
+
+ FSDataInputStream is = null;
+ try {
+ is = fs.open(dictFile);
+ String dictClassName = is.readUTF();
+ Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
+ dict.readFields(is);
+ logger.info("DictionaryProvider read dict from file: " + dictFile);
+ return dict;
+ } finally {
+ IOUtils.closeQuietly(is);
}
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index aded600..ce01eb6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -23,16 +23,11 @@ import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -87,8 +82,8 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
int uhcReducerCount = cube.getConfig().getUHCReducerCount();
int[] uhcIndex = cubeMgr.getUHCIndex(cube.getDescriptor());
- for (int index : uhcIndex) {
- if (index == 1) {
+ for(int index : uhcIndex) {
+ if(index == 1) {
reducerCount += uhcReducerCount - 1;
}
}
@@ -97,6 +92,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
throw new IllegalArgumentException("The max reducer number for FactDistinctColumnsJob is 255, but now it is " + reducerCount + ", decrease 'kylin.engine.mr.uhc-reducer-count'");
}
+
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled);
@@ -121,12 +117,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
attachCubeMetadata(cube, job.getConfiguration());
- /**
- * don't compress the reducer output so that {@link CreateDictionaryJob} and {@link UpdateCubeInfoAfterBuildStep}
- * could read the reducer file directly
- */
- job.getConfiguration().set(BatchConstants.CFG_MAPRED_OUTPUT_COMPRESS, "false");
-
return waitForCompletion(job);
} finally {
@@ -148,22 +138,18 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
private void setupReducer(Path output, int numberOfReducers) throws IOException {
job.setReducerClass(FactDistinctColumnsReducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(Text.class);
job.setPartitionerClass(FactDistinctColumnPartitioner.class);
job.setNumReduceTasks(numberOfReducers);
- //make each reducer output to respective dir
- MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_COLUMN, SequenceFileOutputFormat.class, NullWritable.class, Text.class);
- MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_DICT, SequenceFileOutputFormat.class, NullWritable.class, BytesWritable.class);
- MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_STATISTICS, SequenceFileOutputFormat.class, LongWritable.class, BytesWritable.class);
- MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_PARTITION, TextOutputFormat.class, NullWritable.class, LongWritable.class);
-
-
+ // important, reducer writes HDFS directly at the moment
+ job.setReduceSpeculativeExecution(false);
+
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
- //prevent to create zero-sized default output
- LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
-
deletePath(job.getConfiguration(), output);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5d2fb72..711d991 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -18,25 +18,27 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -45,7 +47,7 @@ import org.apache.kylin.dict.IDictionaryBuilder;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -61,12 +63,14 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
private List<TblColRef> columnList;
+ private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
protected Map<Long, HLLCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
private int samplingPercentage;
+ private List<ByteArray> colValues;
private TblColRef col = null;
private boolean isStatistics = false;
private KylinConfig cubeConfig;
@@ -84,14 +88,10 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
public static final String DICT_FILE_POSTFIX = ".rldict";
public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
- private MultipleOutputs mos;
-
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
- mos = new MultipleOutputs(context);
-
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
@@ -109,20 +109,26 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (collectStatistics && (taskId == numberOfTasks - 1)) {
// hll
isStatistics = true;
+ statisticsOutput = conf.get(BatchConstants.CFG_STATISTICS_OUTPUT);
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
logger.info("Reducer " + taskId + " handling stats");
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
+ isStatistics = false;
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
logger.info("Do not have partition col. This reducer will keep empty");
}
+ colValues = Lists.newLinkedList();
+ logger.info("Reducer " + taskId + " handling partition column " + col);
} else {
// normal col
+ isStatistics = false;
col = columnList.get(reducerIdToColumnIndex.get(taskId));
+ colValues = Lists.newLinkedList();
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
@@ -188,13 +194,15 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
logAFewRows(value);
builder.addValue(value);
} else {
- byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
- // output written to baseDir/colName/-r-00000 (etc)
- String fileName = col.getName() + "/";
- mos.write(BatchConstants.CFG_OUTPUT_COLUMN, NullWritable.get(), new Text(keyBytes), fileName);
+ colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
+ if (colValues.size() == 1000000) { //spill every 1 million
+ logger.info("spill values to disk...");
+ outputDistinctValues(col, colValues, context);
+ colValues.clear();
+ }
}
}
-
+
rowCount++;
}
@@ -204,104 +212,162 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
- @Override
- protected void doCleanup(Context context) throws IOException, InterruptedException {
- if (isStatistics) {
- //output the hll info;
- List<Long> allCuboids = Lists.newArrayList();
- allCuboids.addAll(cuboidHLLMap.keySet());
- Collections.sort(allCuboids);
+ private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
+ final Configuration conf = context.getConfiguration();
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
+ final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
+ final Path colDir = new Path(outputPath, col.getIdentity());
+ final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
+ final Path outputFile = new Path(colDir, fileName);
+
+ FSDataOutputStream out = null;
+ try {
+ if (!fs.exists(colDir)) {
+ fs.mkdirs(colDir);
+ }
- logMapperAndCuboidStatistics(allCuboids); // for human check
- outputStatistics(allCuboids);
- } else if (isPartitionCol) {
- // partition col
- outputPartitionInfo();
- } else {
- // normal col
- if (isReducerLocalBuildDict) {
- Dictionary<String> dict = builder.build();
- outputDict(col, dict);
+ if (fs.exists(outputFile)) {
+ out = fs.append(outputFile);
+ logger.info("append file " + outputFile);
+ } else {
+ out = fs.create(outputFile);
+ logger.info("create file " + outputFile);
}
- }
- mos.close();
+ for (ByteArray value : values) {
+ out.write(value.array(), value.offset(), value.length());
+ out.write('\n');
+ }
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
}
- private void outputPartitionInfo() throws IOException, InterruptedException {
- if (col != null) {
- // output written to baseDir/colName/colName.pci-r-00000 (etc)
- String partitionFileName = col.getName() + "/" + col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
+ private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
+ final String fileName = col.getIdentity() + DICT_FILE_POSTFIX;
+ FSDataOutputStream out = getOutputStream(context, fileName);
+ try {
+ String dictClassName = dict.getClass().getName();
+ out.writeUTF(dictClassName);
+ dict.write(out);
+ logger.info("reducer id is:+" + taskId + " colName:" + col.getName() + " writing dict at file : " + fileName + " dict class:" + dictClassName);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
- mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMinValue), partitionFileName);
- mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMaxValue), partitionFileName);
+ private void outputPartitionInfo(Context context) throws IOException {
+ final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX;
+ FSDataOutputStream out = getOutputStream(context, fileName);
+ try {
+ out.writeLong(timeMinValue);
+ out.writeLong(timeMaxValue);
logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
+ final Configuration conf = context.getConfiguration();
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
+ final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
+ final Path outputFile = new Path(outputPath, outputFileName);
+ if (!fs.exists(outputPath)) {
+ fs.mkdirs(outputPath);
}
+ FSDataOutputStream out = fs.create(outputFile);
+ return out;
}
- private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
- // output written to baseDir/colName/colName.rldict-r-00000 (etc)
- String dictFileName = col.getName() + "/" + col.getName() + DICT_FILE_POSTFIX;
+ @Override
+ protected void doCleanup(Context context) throws IOException, InterruptedException {
+ if (isStatistics) {
+ // output the hll info
+ long grandTotal = 0;
+ for (HLLCounter hll : cuboidHLLMap.values()) {
+ grandTotal += hll.getCountEstimate();
+ }
+ double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) {
- outputStream.writeUTF(dict.getClass().getName());
- dict.write(outputStream);
+ int mapperNumber = baseCuboidRowCountInMappers.size();
- mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new BytesWritable(baos.toByteArray()), dictFileName);
+ writeMapperAndCuboidStatistics(context); // for human check
+ CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
+ cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
+ } else if (isPartitionCol) {
+ // partition col
+ if (col != null) {
+ outputPartitionInfo(context);
+ }
+ } else {
+ // normal col
+ if (isReducerLocalBuildDict) {
+ Dictionary<String> dict = builder.build();
+ outputDict(col, dict, context);
+ } else {
+ if (colValues.size() > 0) {
+ outputDistinctValues(col, colValues, context);
+ colValues.clear();
+ }
+ }
}
}
- private void outputStatistics(List<Long> allCuboids) throws IOException, InterruptedException {
- // output written to baseDir/statistics/statistics-r-00000 (etc)
- String statisticsFileName = BatchConstants.CFG_OUTPUT_STATISTICS + "/" + BatchConstants.CFG_OUTPUT_STATISTICS;
+ private void writeMapperAndCuboidStatistics(Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
+ Path path = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME);
+ FSDataOutputStream out = fs.create(path);
- ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
+ try {
+ String msg;
- // mapper overlap ratio at key -1
- long grandTotal = 0;
- for (HLLCounter hll : cuboidHLLMap.values()) {
- grandTotal += hll.getCountEstimate();
- }
- double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
- mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-1), new BytesWritable(Bytes.toBytes(mapperOverlapRatio)), statisticsFileName);
+ List<Long> allCuboids = Lists.newArrayList();
+ allCuboids.addAll(cuboidHLLMap.keySet());
+ Collections.sort(allCuboids);
- // mapper number at key -2
- mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-2), new BytesWritable(Bytes.toBytes(baseCuboidRowCountInMappers.size())), statisticsFileName);
+ msg = "Total cuboid number: \t" + allCuboids.size();
+ writeLine(out, msg);
+ msg = "Samping percentage: \t" + samplingPercentage;
+ writeLine(out, msg);
+
+ writeLine(out, "The following statistics are collected based on sampling data.");
+ writeLine(out, "Number of Mappers: " + baseCuboidRowCountInMappers.size());
+ for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
+ if (baseCuboidRowCountInMappers.get(i) > 0) {
+ msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
+ writeLine(out, msg);
+ }
+ }
- // sampling percentage at key 0
- mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(0L), new BytesWritable(Bytes.toBytes(samplingPercentage)), statisticsFileName);
+ long grantTotal = 0;
+ for (long i : allCuboids) {
+ grantTotal += cuboidHLLMap.get(i).getCountEstimate();
+ msg = "Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate();
+ writeLine(out, msg);
+ }
- for (long i : allCuboids) {
- valueBuf.clear();
- cuboidHLLMap.get(i).writeRegisters(valueBuf);
- valueBuf.flip();
- mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()), statisticsFileName);
- }
- }
+ msg = "Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge;
+ writeLine(out, msg);
- private void logMapperAndCuboidStatistics(List<Long> allCuboids) throws IOException {
- logger.info("Total cuboid number: \t" + allCuboids.size());
- logger.info("Samping percentage: \t" + samplingPercentage);
- logger.info("The following statistics are collected based on sampling data.");
- logger.info("Number of Mappers: " + baseCuboidRowCountInMappers.size());
+ msg = "After merge, the cube has row count: \t " + grantTotal;
+ writeLine(out, msg);
- for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
- if (baseCuboidRowCountInMappers.get(i) > 0) {
- logger.info("Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i));
+ if (grantTotal > 0) {
+ msg = "The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal;
+ writeLine(out, msg);
}
- }
- long grantTotal = 0;
- for (long i : allCuboids) {
- grantTotal += cuboidHLLMap.get(i).getCountEstimate();
- logger.info("Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate());
+ } finally {
+ IOUtils.closeQuietly(out);
}
+ }
+
+ private void writeLine(FSDataOutputStream out, String msg) throws IOException {
+ out.write(msg.getBytes());
+ out.write('\n');
- logger.info("Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge);
- logger.info("After merge, the cube has row count: \t " + grantTotal);
- if (grantTotal > 0) {
- logger.info("The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal);
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 28f99fb..2671042 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -60,11 +60,9 @@ public class SaveStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
FileSystem fs = HadoopUtil.getWorkingFileSystem();
- Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
- Path statisticsFilePath = HadoopUtil.getFilterOnlyPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS);
- if (statisticsFilePath == null) {
- throw new IOException("fail to find the statistics file in base dir: " + statisticsDir);
- }
+ Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
+ if (!fs.exists(statisticsFilePath))
+ throw new IOException("File " + statisticsFilePath + " does not exists");
FSDataInputStream is = fs.open(statisticsFilePath);
try {
@@ -112,7 +110,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
+
// in-mem cubing is good when
// 1) the cluster has enough mapper slots to run in parallel
// 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 81d5c42..dc80399 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -80,13 +80,8 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
- FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
- Path outputFile = HadoopUtil.getFilterOnlyPath(fs, colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
- if (outputFile == null) {
- throw new IOException("fail to find the partition file in base dir: " + colDir);
- }
-
+ Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
@@ -102,4 +97,5 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
segment.setDateRangeStart(minValue);
segment.setDateRangeEnd(maxValue);
}
+
}
[18/47] kylin git commit: KYLIN-2412 Unclosed DataOutputStream in
RoaringBitmapCounter#write()
Posted by bi...@apache.org.
KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d264339b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d264339b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d264339b
Branch: refs/heads/KYLIN-2361
Commit: d264339b1c16c195ffafc2217b793d81bdbd6434
Parents: 7576a09
Author: gaodayue <ga...@meituan.com>
Authored: Sun Jan 22 14:02:09 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Sun Jan 22 14:02:09 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d264339b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..fb9dcfc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -107,7 +107,9 @@ public class RoaringBitmapCounter implements BitmapCounter {
if (out.remaining() < bitmap.serializedSizeInBytes()) {
throw new BufferOverflowException();
}
- bitmap.serialize(new DataOutputStream(new ByteBufferOutputStream(out)));
+ try (DataOutputStream dos = new DataOutputStream(new ByteBufferOutputStream(out))) {
+ bitmap.serialize(dos);
+ }
}
@Override
[41/47] kylin git commit: KYLIN-2414 Distinguish UHC columns from
normal columns in KYLIN-2217
Posted by bi...@apache.org.
KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b
Branch: refs/heads/KYLIN-2361
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <95...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 26 09:56:54 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 6 ++++++
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements Serializable {
}
public boolean isReducerLocalBuildDict() {
- if (getUHCReducerCount() != 1) {
- return false;
- }
return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
isReducerLocalBuildDict = false;
}
+ if(config.getUHCReducerCount() > 1) {
+ int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+ int colIndex = reducerIdToColumnIndex.get(taskId);
+ if (uhcIndex[colIndex] == 1)
+ isReducerLocalBuildDict = false; //for UHC columns, this feature should be disabled
+ }
if (isReducerLocalBuildDict) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
[03/47] kylin git commit: KYLIN-2410 remove force qualified path in
HadoopUtil.getFileSystem
Posted by bi...@apache.org.
KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61833d95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61833d95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61833d95
Branch: refs/heads/KYLIN-2361
Commit: 61833d952daa3ed7e3938f6e310caca5a52a4974
Parents: eddb695
Author: Yang Li <li...@apache.org>
Authored: Thu Jan 19 10:35:50 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Jan 19 12:42:44 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/util/HadoopUtil.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/61833d95/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 390c209..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -82,9 +82,6 @@ public class HadoopUtil {
}
public static FileSystem getFileSystem(Path path, Configuration conf) {
- if (StringUtils.isBlank(path.toUri().getScheme()))
- throw new IllegalArgumentException("Path must be qualified: " + path);
-
try {
return path.getFileSystem(conf);
} catch (IOException e) {
[44/47] kylin git commit: KYLIN-2418 drop unused useSandbox
Posted by bi...@apache.org.
KYLIN-2418 drop unused useSandbox
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/546f88f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/546f88f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/546f88f3
Branch: refs/heads/KYLIN-2361
Commit: 546f88f38ca017773c17ccaf2cf4812b0c7c4666
Parents: 0582512
Author: Yang Li <li...@apache.org>
Authored: Mon Jan 30 09:07:29 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Jan 31 08:11:41 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/FileResourceStore.java | 4 ++++
.../org/apache/kylin/common/persistence/ResourceTool.java | 10 ++++++----
kylin-it/pom.xml | 6 ------
pom.xml | 8 --------
.../apache/kylin/storage/hbase/HBaseResourceStore.java | 5 +++++
5 files changed, 15 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index dcd3f38..3e012f5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -160,4 +160,8 @@ public class FileResourceStore extends ResourceStore {
return new File(root, resPath);
}
+ @Override
+ public String toString() {
+ return root.getAbsolutePath();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b3aac09..6ba68ae 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -133,22 +133,24 @@ public class ResourceTool {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
+ logger.info("Copy from {} to {}", src, dst);
+
copyR(src, dst, path);
}
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+ logger.info("Copy from {} to {}", src, dst);
+
for (String path : paths) {
copyR(src, dst, path);
}
}
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
-
- ResourceStore src = ResourceStore.getStore(srcConfig);
- ResourceStore dst = ResourceStore.getStore(dstConfig);
- copyR(src, dst, "/");
+ copy(srcConfig, dstConfig, "/");
}
public static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f88db9f..9662806 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -272,10 +272,6 @@
</excludes>
<systemProperties>
<property>
- <name>useSandbox</name>
- <value>true</value>
- </property>
- <property>
<name>log4j.configuration</name>
<value>file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</value>
</property>
@@ -298,7 +294,6 @@
<classpathScope>test</classpathScope>
<executable>java</executable>
<arguments>
- <argument>-DuseSandbox=true</argument>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
<argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
@@ -320,7 +315,6 @@
<classpathScope>test</classpathScope>
<executable>java</executable>
<arguments>
- <argument>-DuseSandbox=true</argument>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
<argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bd1882e..bf33e07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1159,10 +1159,6 @@
</excludes>
<systemProperties>
<property>
- <name>useSandbox</name>
- <value>false</value>
- </property>
- <property>
<name>buildCubeUsingProvidedData</name>
<value>false</value>
</property>
@@ -1263,10 +1259,6 @@
</excludes>
<systemProperties>
<property>
- <name>useSandbox</name>
- <value>false</value>
- </property>
- <property>
<name>buildCubeUsingProvidedData</name>
<value>false</value>
</property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 501f1e4..74ab017 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -371,4 +371,9 @@ public class HBaseResourceStore extends ResourceStore {
return put;
}
+
+ @Override
+ public String toString() {
+ return getAllInOneTableName() + "@hbase";
+ }
}
[05/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and
Avatica to 1.9.0
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
*/
package org.apache.calcite.sql2rel;
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
import org.apache.calcite.avatica.util.Spaces;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDelete;
import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
/*
* The code has synced with calcite. Hope one day, we could remove the hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
public class SqlToRelConverter {
//~ Static fields/initializers ---------------------------------------------
+ protected static final Logger SQL2REL_LOGGER =
+ CalciteTrace.getSqlToRelTracer();
+
+ private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
/** Size of the smallest IN list that will be converted to a semijoin to a
* static table. */
- public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
- protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
- private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+ public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+ @Deprecated // to be removed before 2.0
+ public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+ DEFAULT_IN_SUB_QUERY_THRESHOLD;
//~ Instance fields --------------------------------------------------------
- public final SqlToRelConverter.Config config;
- public final RelOptTable.ViewExpander viewExpander;
+
protected final SqlValidator validator;
protected final RexBuilder rexBuilder;
protected final Prepare.CatalogReader catalogReader;
protected final RelOptCluster cluster;
+ private DefaultValueFactory defaultValueFactory;
+ private SubQueryConverter subQueryConverter;
protected final List<RelNode> leaves = new ArrayList<>();
- protected final RelDataTypeFactory typeFactory;
private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
private final SqlOperatorTable opTab;
+ protected final RelDataTypeFactory typeFactory;
private final SqlNodeToRexConverter exprConverter;
+ private int explainParamCount;
+ public final SqlToRelConverter.Config config;
+
/**
- * Fields used in name resolution for correlated subqueries.
+ * Fields used in name resolution for correlated sub-queries.
*/
- private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
+ private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred =
+ new HashMap<>();
+
/**
* Stack of names of datasets requested by the <code>
* TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
*/
private final Deque<String> datasetStack = new ArrayDeque<>();
+
/**
- * Mapping of non-correlated subqueries that have been converted to their
- * equivalent constants. Used to avoid re-evaluating the subquery if it's
+ * Mapping of non-correlated sub-queries that have been converted to their
+ * equivalent constants. Used to avoid re-evaluating the sub-query if it's
* already been evaluated.
*/
- private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
- private int explainParamCount;
+ private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
+ new HashMap<>();
+
+ public final RelOptTable.ViewExpander viewExpander;
//~ Constructors -----------------------------------------------------------
/**
@@ -258,23 +274,46 @@ public class SqlToRelConverter {
* @param convertletTable Expression converter
*/
@Deprecated // to be removed before 2.0
- public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable) {
- this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable, Config.DEFAULT);
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptPlanner planner,
+ RexBuilder rexBuilder,
+ SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader,
+ RelOptCluster.create(planner, rexBuilder), convertletTable,
+ Config.DEFAULT);
}
@Deprecated // to be removed before 2.0
- public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable) {
- this(viewExpander, validator, catalogReader, cluster, convertletTable, Config.DEFAULT);
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader, cluster, convertletTable,
+ Config.DEFAULT);
}
/* Creates a converter. */
- public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable, Config config) {
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable,
+ Config config) {
this.viewExpander = viewExpander;
- this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable();
+ this.opTab =
+ (validator
+ == null) ? SqlStdOperatorTable.instance()
+ : validator.getOperatorTable();
this.validator = validator;
this.catalogReader = catalogReader;
this.defaultValueFactory = new NullDefaultValueFactory();
- this.subqueryConverter = new NoOpSubqueryConverter();
+ this.subQueryConverter = new NoOpSubQueryConverter();
this.rexBuilder = cluster.getRexBuilder();
this.typeFactory = rexBuilder.getTypeFactory();
this.cluster = Preconditions.checkNotNull(cluster);
@@ -285,157 +324,6 @@ public class SqlToRelConverter {
//~ Methods ----------------------------------------------------------------
- private static boolean isStream(SqlNode query) {
- return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
- }
-
- public static boolean isOrdered(SqlNode query) {
- switch (query.getKind()) {
- case SELECT:
- return ((SqlSelect) query).getOrderList() != null && ((SqlSelect) query).getOrderList().size() > 0;
- case WITH:
- return isOrdered(((SqlWith) query).body);
- case ORDER_BY:
- return ((SqlOrderBy) query).orderList.size() > 0;
- default:
- return false;
- }
- }
-
- /**
- * Returns whether a given node contains a {@link SqlInOperator}.
- *
- * @param node a RexNode tree
- */
- private static boolean containsInOperator(SqlNode node) {
- try {
- SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
- }
- };
- node.accept(visitor);
- return false;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return true;
- }
- }
-
- /**
- * Push down all the NOT logical operators into any IN/NOT IN operators.
- *
- * @param sqlNode the root node from which to look for NOT operators
- * @return the transformed SqlNode representation with NOT pushed down.
- */
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
- if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
- SqlCall sqlCall = (SqlCall) sqlNode;
- if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
- SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
- for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
- }
- return sqlNode;
- } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode childNode = sqlCall.operand(0);
- assert childNode instanceof SqlCall;
- SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
- if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
- SqlNode[] andOperands = childSqlCall.getOperands();
- SqlNode[] orOperands = new SqlNode[andOperands.length];
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]);
- }
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
- }
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
- SqlNode[] orOperands = childSqlCall.getOperands();
- SqlNode[] andOperands = new SqlNode[orOperands.length];
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]);
- }
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
- }
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode[] notOperands = childSqlCall.getOperands();
- assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
- } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
- SqlNode[] inOperands = childSqlCall.getOperands();
- SqlInOperator inOp = (SqlInOperator) childSqlCall.getOperator();
- if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
- } else {
- return SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
- }
- } else {
- // childSqlCall is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // sqlNode is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // tree rooted at sqlNode does not contain inOperator
- return sqlNode;
- }
- }
-
- private static boolean containsNullLiteral(SqlNodeList valueList) {
- for (SqlNode node : valueList.getList()) {
- if (node instanceof SqlLiteral) {
- SqlLiteral lit = (SqlLiteral) node;
- if (lit.getValue() == null) {
- return true;
- }
- }
- }
- return false;
- }
-
- private static JoinRelType convertJoinType(JoinType joinType) {
- switch (joinType) {
- case COMMA:
- case INNER:
- case CROSS:
- return JoinRelType.INNER;
- case FULL:
- return JoinRelType.FULL;
- case LEFT:
- return JoinRelType.LEFT;
- case RIGHT:
- return JoinRelType.RIGHT;
- default:
- throw Util.unexpected(joinType);
- }
- }
-
- private static boolean desc(RelFieldCollation.Direction direction) {
- switch (direction) {
- case DESCENDING:
- case STRICTLY_DESCENDING:
- return true;
- default:
- return false;
- }
- }
-
- /** Creates a builder for a {@link Config}. */
- public static ConfigBuilder configBuilder() {
- return new ConfigBuilder();
- }
-
/**
* @return the RelOptCluster in use.
*/
@@ -490,7 +378,7 @@ public class SqlToRelConverter {
}
/**
- * @return mapping of non-correlated subqueries that have been converted to
+ * @return mapping of non-correlated sub-queries that have been converted to
* the constants that they evaluate to
*/
public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
@@ -498,13 +386,14 @@ public class SqlToRelConverter {
}
/**
- * Adds to the current map of non-correlated converted subqueries the
- * elements from another map that contains non-correlated subqueries that
+ * Adds to the current map of non-correlated converted sub-queries the
+ * elements from another map that contains non-correlated sub-queries that
* have been converted by another SqlToRelConverter.
*
* @param alreadyConvertedNonCorrSubqs the other map
*/
- public void addConvertedNonCorrSubqs(Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+ public void addConvertedNonCorrSubqs(
+ Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
}
@@ -519,13 +408,13 @@ public class SqlToRelConverter {
}
/**
- * Sets a new SubqueryConverter. To have any effect, this must be called
+ * Sets a new SubQueryConverter. To have any effect, this must be called
* before any convert method.
*
- * @param converter new SubqueryConverter
+ * @param converter new SubQueryConverter
*/
- public void setSubqueryConverter(SubqueryConverter converter) {
- subqueryConverter = converter;
+ public void setSubQueryConverter(SubQueryConverter converter) {
+ subQueryConverter = converter;
}
/**
@@ -548,24 +437,42 @@ public class SqlToRelConverter {
// SQL statement is something like an INSERT which has no
// validator type information associated with its result,
// hence the namespace check above.)
- final List<RelDataTypeField> validatedFields = validator.getValidatedNodeType(query).getFieldList();
- final RelDataType validatedRowType = validator.getTypeFactory().createStructType(Pair.right(validatedFields), SqlValidatorUtil.uniquify(Pair.left(validatedFields), catalogReader.isCaseSensitive()));
-
- final List<RelDataTypeField> convertedFields = result.getRowType().getFieldList().subList(0, validatedFields.size());
- final RelDataType convertedRowType = validator.getTypeFactory().createStructType(convertedFields);
-
- if (!RelOptUtil.equal("validated row type", validatedRowType, "converted row type", convertedRowType, Litmus.IGNORE)) {
- throw new AssertionError("Conversion to relational algebra failed to " + "preserve datatypes:\n" + "validated type:\n" + validatedRowType.getFullTypeString() + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n" + RelOptUtil.toString(result));
+ final List<RelDataTypeField> validatedFields =
+ validator.getValidatedNodeType(query).getFieldList();
+ final RelDataType validatedRowType =
+ validator.getTypeFactory().createStructType(
+ Pair.right(validatedFields),
+ SqlValidatorUtil.uniquify(Pair.left(validatedFields),
+ catalogReader.isCaseSensitive()));
+
+ final List<RelDataTypeField> convertedFields =
+ result.getRowType().getFieldList().subList(0, validatedFields.size());
+ final RelDataType convertedRowType =
+ validator.getTypeFactory().createStructType(convertedFields);
+
+ if (!RelOptUtil.equal("validated row type", validatedRowType,
+ "converted row type", convertedRowType, Litmus.IGNORE)) {
+ throw new AssertionError("Conversion to relational algebra failed to "
+ + "preserve datatypes:\n"
+ + "validated type:\n"
+ + validatedRowType.getFullTypeString()
+ + "\nconverted type:\n"
+ + convertedRowType.getFullTypeString()
+ + "\nrel:\n"
+ + RelOptUtil.toString(result));
}
}
- public RelNode flattenTypes(RelNode rootRel, boolean restructure) {
- RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
- return typeFlattener.rewrite(rootRel, restructure);
+ public RelNode flattenTypes(
+ RelNode rootRel,
+ boolean restructure) {
+ RelStructuredTypeFlattener typeFlattener =
+ new RelStructuredTypeFlattener(rexBuilder, createToRelContext(), restructure);
+ return typeFlattener.rewrite(rootRel);
}
/**
- * If subquery is correlated and decorrelation is enabled, performs
+ * If sub-query is correlated and decorrelation is enabled, performs
* decorrelation.
*
* @param query Query
@@ -606,14 +513,21 @@ public class SqlToRelConverter {
// Trim fields that are not used by their consumer.
if (isTrimUnusedFields()) {
final RelFieldTrimmer trimmer = newFieldTrimmer();
- final List<RelCollation> collations = rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
+ final List<RelCollation> collations =
+ rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
rootRel = trimmer.trim(rootRel);
- if (!ordered && collations != null && !collations.isEmpty() && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
- final RelTraitSet traitSet = rootRel.getTraitSet().replace(RelCollationTraitDef.INSTANCE, collations);
+ if (!ordered
+ && collations != null
+ && !collations.isEmpty()
+ && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
+ final RelTraitSet traitSet = rootRel.getTraitSet()
+ .replace(RelCollationTraitDef.INSTANCE, collations);
rootRel = rootRel.copy(traitSet, rootRel.getInputs());
}
if (SQL2REL_LOGGER.isDebugEnabled()) {
- SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ SQL2REL_LOGGER.debug(
+ RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel,
+ SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
}
return rootRel;
@@ -625,7 +539,8 @@ public class SqlToRelConverter {
* @return Field trimmer
*/
protected RelFieldTrimmer newFieldTrimmer() {
- final RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster, null);
+ final RelBuilder relBuilder =
+ RelFactories.LOGICAL_BUILDER.create(cluster, null);
return new RelFieldTrimmer(validator, relBuilder);
}
@@ -640,14 +555,18 @@ public class SqlToRelConverter {
* will become a JDBC result set; <code>false</code> if
* the query will be part of a view.
*/
- public RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top) {
+ public RelRoot convertQuery(
+ SqlNode query,
+ final boolean needsValidation,
+ final boolean top) {
SqlNode origQuery = query; /* OVERRIDE POINT */
-
+
if (needsValidation) {
query = validator.validate(query);
}
- RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(cluster.getMetadataProvider()));
+ RelMetadataQuery.THREAD_PROVIDERS.set(
+ JaninoRelMetadataProvider.of(cluster.getMetadataProvider()));
RelNode result = convertQueryRecursive(query, top, null).rel;
if (top) {
if (isStream(query)) {
@@ -663,18 +582,23 @@ public class SqlToRelConverter {
checkConvertedType(query, result);
if (SQL2REL_LOGGER.isDebugEnabled()) {
- SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode", result, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ SQL2REL_LOGGER.debug(
+ RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
+ result, SqlExplainFormat.TEXT,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
final RelDataType validatedRowType = validator.getValidatedNodeType(query);
- return hackSelectStar(origQuery, RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation));
+ RelRoot origResult = RelRoot.of(result, validatedRowType, query.getKind())
+ .withCollation(collation);
+ return hackSelectStar(origQuery, origResult);
}
/* OVERRIDE POINT */
private RelRoot hackSelectStar(SqlNode query, RelRoot root) {
/*
* Rel tree is like:
- *
+ *
* LogicalSort (optional)
* |- LogicalProject
* |- LogicalFilter (optional)
@@ -690,7 +614,7 @@ public class SqlToRelConverter {
} else {
return root;
}
-
+
RelNode input = rootPrj.getInput();
if (!(//
input.getClass().getSimpleName().equals("OLAPTableScan")//
@@ -700,13 +624,13 @@ public class SqlToRelConverter {
if (rootPrj.getRowType().getFieldCount() < input.getRowType().getFieldCount())
return root;
-
+
RelDataType inType = rootPrj.getRowType();
List<String> inFields = inType.getFieldNames();
List<RexNode> projExp = new ArrayList<>();
List<Pair<Integer, String>> projFields = new ArrayList<>();
- FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder();
- FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder();
+ RelDataTypeFactory.FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder();
+ RelDataTypeFactory.FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder();
for (int i = 0; i < inFields.size(); i++) {
if (!inFields.get(i).startsWith("_KY_")) {
projExp.add(rootPrj.getProjects().get(i));
@@ -721,15 +645,34 @@ public class SqlToRelConverter {
if (rootSort != null) {
rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
}
-
+
RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder);
root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, root.collation);
-
+
validator.setValidatedNodeType(query, validRowType);
-
+
return root;
}
+ private static boolean isStream(SqlNode query) {
+ return query instanceof SqlSelect
+ && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ }
+
+ public static boolean isOrdered(SqlNode query) {
+ switch (query.getKind()) {
+ case SELECT:
+ return ((SqlSelect) query).getOrderList() != null
+ && ((SqlSelect) query).getOrderList().size() > 0;
+ case WITH:
+ return isOrdered(((SqlWith) query).body);
+ case ORDER_BY:
+ return ((SqlOrderBy) query).orderList.size() > 0;
+ default:
+ return false;
+ }
+ }
+
private RelCollation requiredCollation(RelNode r) {
if (r instanceof Sort) {
return ((Sort) r).collation;
@@ -756,7 +699,8 @@ public class SqlToRelConverter {
/**
* Factory method for creating translation workspace.
*/
- protected Blackboard createBlackboard(SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap, boolean top) {
+ protected Blackboard createBlackboard(SqlValidatorScope scope,
+ Map<String, RexNode> nameToNodeMap, boolean top) {
return new Blackboard(scope, nameToNodeMap, top);
}
@@ -764,25 +708,45 @@ public class SqlToRelConverter {
* Implementation of {@link #convertSelect(SqlSelect, boolean)};
* derived class may override.
*/
- protected void convertSelectImpl(final Blackboard bb, SqlSelect select) {
- convertFrom(bb, select.getFrom());
- convertWhere(bb, select.getWhere());
+ protected void convertSelectImpl(
+ final Blackboard bb,
+ SqlSelect select) {
+ convertFrom(
+ bb,
+ select.getFrom());
+ convertWhere(
+ bb,
+ select.getWhere());
final List<SqlNode> orderExprList = new ArrayList<>();
final List<RelFieldCollation> collationList = new ArrayList<>();
- gatherOrderExprs(bb, select, select.getOrderList(), orderExprList, collationList);
- final RelCollation collation = cluster.traitSet().canonize(RelCollations.of(collationList));
+ gatherOrderExprs(
+ bb,
+ select,
+ select.getOrderList(),
+ orderExprList,
+ collationList);
+ final RelCollation collation =
+ cluster.traitSet().canonize(RelCollations.of(collationList));
if (validator.isAggregate(select)) {
- convertAgg(bb, select, orderExprList);
+ convertAgg(
+ bb,
+ select,
+ orderExprList);
} else {
- convertSelectList(bb, select, orderExprList);
+ convertSelectList(
+ bb,
+ select,
+ orderExprList);
}
if (select.isDistinct()) {
distinctify(bb, true);
}
- convertOrder(select, bb, collation, orderExprList, select.getOffset(), select.getFetch());
+ convertOrder(
+ select, bb, collation, orderExprList, select.getOffset(),
+ select.getFetch());
bb.setRoot(bb.root, true);
}
@@ -798,7 +762,9 @@ public class SqlToRelConverter {
* @param bb Blackboard
* @param checkForDupExprs Check for duplicate expressions
*/
- private void distinctify(Blackboard bb, boolean checkForDupExprs) {
+ private void distinctify(
+ Blackboard bb,
+ boolean checkForDupExprs) {
// Look for duplicate expressions in the project.
// Say we have 'select x, y, x, z'.
// Then dups will be {[2, 0]}
@@ -832,7 +798,9 @@ public class SqlToRelConverter {
newProjects.add(RexInputRef.of2(i, fields));
}
}
- rel = LogicalProject.create(rel, Pair.left(newProjects), Pair.right(newProjects));
+ rel =
+ LogicalProject.create(rel, Pair.left(newProjects),
+ Pair.right(newProjects));
bb.root = rel;
distinctify(bb, false);
rel = bb.root;
@@ -843,21 +811,34 @@ public class SqlToRelConverter {
for (int i = 0; i < fields.size(); i++) {
final int origin = origins.get(i);
RelDataTypeField field = fields.get(i);
- undoProjects.add(Pair.of((RexNode) new RexInputRef(squished.get(origin), field.getType()), field.getName()));
+ undoProjects.add(
+ Pair.of(
+ (RexNode) new RexInputRef(
+ squished.get(origin), field.getType()),
+ field.getName()));
}
- rel = LogicalProject.create(rel, Pair.left(undoProjects), Pair.right(undoProjects));
- bb.setRoot(rel, false);
+ rel =
+ LogicalProject.create(rel, Pair.left(undoProjects),
+ Pair.right(undoProjects));
+ bb.setRoot(
+ rel,
+ false);
return;
}
// Usual case: all of the expressions in the SELECT clause are
// different.
- final ImmutableBitSet groupSet = ImmutableBitSet.range(rel.getRowType().getFieldCount());
- rel = createAggregate(bb, false, groupSet, ImmutableList.of(groupSet), ImmutableList.<AggregateCall> of());
-
- bb.setRoot(rel, false);
+ final ImmutableBitSet groupSet =
+ ImmutableBitSet.range(rel.getRowType().getFieldCount());
+ rel =
+ createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
+ ImmutableList.<AggregateCall>of());
+
+ bb.setRoot(
+ rel,
+ false);
}
private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
@@ -882,16 +863,29 @@ public class SqlToRelConverter {
* returning first row
* @param fetch Expression for number of rows to fetch
*/
- protected void convertOrder(SqlSelect select, Blackboard bb, RelCollation collation, List<SqlNode> orderExprList, SqlNode offset, SqlNode fetch) {
- if (select.getOrderList() == null || select.getOrderList().getList().isEmpty()) {
+ protected void convertOrder(
+ SqlSelect select,
+ Blackboard bb,
+ RelCollation collation,
+ List<SqlNode> orderExprList,
+ SqlNode offset,
+ SqlNode fetch) {
+ if (select.getOrderList() == null
+ || select.getOrderList().getList().isEmpty()) {
assert collation.getFieldCollations().isEmpty();
- if ((offset == null || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO)) && fetch == null) {
+ if ((offset == null
+ || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
+ && fetch == null) {
return;
}
}
// Create a sorter using the previously constructed collations.
- bb.setRoot(LogicalSort.create(bb.root, collation, offset == null ? null : convertExpression(offset), fetch == null ? null : convertExpression(fetch)), false);
+ bb.setRoot(
+ LogicalSort.create(bb.root, collation,
+ offset == null ? null : convertExpression(offset),
+ fetch == null ? null : convertExpression(fetch)),
+ false);
// If extra expressions were added to the project list for sorting,
// add another project to remove them. But make the collation empty, because
@@ -901,300 +895,492 @@ public class SqlToRelConverter {
if (orderExprList.size() > 0 && !bb.top) {
final List<RexNode> exprs = new ArrayList<>();
final RelDataType rowType = bb.root.getRowType();
- final int fieldCount = rowType.getFieldCount() - orderExprList.size();
+ final int fieldCount =
+ rowType.getFieldCount() - orderExprList.size();
for (int i = 0; i < fieldCount; i++) {
exprs.add(rexBuilder.makeInputRef(bb.root, i));
}
- bb.setRoot(LogicalProject.create(bb.root, exprs, rowType.getFieldNames().subList(0, fieldCount)), false);
+ bb.setRoot(
+ LogicalProject.create(bb.root, exprs,
+ rowType.getFieldNames().subList(0, fieldCount)),
+ false);
}
}
/**
- * Converts a WHERE clause.
+ * Returns whether a given node contains a {@link SqlInOperator}.
*
- * @param bb Blackboard
- * @param where WHERE clause, may be null
+ * @param node a RexNode tree
*/
- private void convertWhere(final Blackboard bb, final SqlNode where) {
- if (where == null) {
- return;
- }
- SqlNode newWhere = pushDownNotForIn(where);
- replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- final RexNode convertedWhere = bb.convertExpression(newWhere);
-
- // only allocate filter if the condition is not TRUE
- if (convertedWhere.isAlwaysTrue()) {
- return;
- }
-
- final RelNode filter = RelOptUtil.createFilter(bb.root, convertedWhere);
- final RelNode r;
- final CorrelationUse p = getCorrelationUse(bb, filter);
- if (p != null) {
- assert p.r instanceof Filter;
- Filter f = (Filter) p.r;
- r = LogicalFilter.create(f.getInput(), f.getCondition(), ImmutableSet.of(p.id));
- } else {
- r = filter;
- }
-
- bb.setRoot(r, false);
- }
-
- private void replaceSubqueries(final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) {
- findSubqueries(bb, expr, logic, false);
- for (SubQuery node : bb.subqueryList) {
- substituteSubquery(bb, node);
+ private static boolean containsInOperator(
+ SqlNode node) {
+ try {
+ SqlVisitor<Void> visitor =
+ new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visit(call);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
}
}
- private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
- final RexNode expr = subQuery.expr;
- if (expr != null) {
- // Already done.
- return;
- }
-
- final SqlBasicCall call;
- final RelNode rel;
- final SqlNode query;
- final Pair<RelNode, Boolean> converted;
- switch (subQuery.node.getKind()) {
- case CURSOR:
- convertCursor(bb, subQuery);
- return;
-
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case ARRAY_QUERY_CONSTRUCTOR:
- rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
- subQuery.expr = bb.register(rel, JoinRelType.INNER);
- return;
-
- case IN:
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(1);
- if (!config.isExpand() && !(query instanceof SqlNodeList)) {
- return;
- }
- final SqlNode leftKeyNode = call.operand(0);
-
- final List<RexNode> leftKeys;
- switch (leftKeyNode.getKind()) {
- case ROW:
- leftKeys = Lists.newArrayList();
- for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
- leftKeys.add(bb.convertExpression(sqlExpr));
- }
- break;
- default:
- leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
- }
-
- final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
- if (query instanceof SqlNodeList) {
- SqlNodeList valueList = (SqlNodeList) query;
- if (!containsNullLiteral(valueList) && valueList.size() < getInSubqueryThreshold()) {
- // We're under the threshold, so convert to OR.
- subQuery.expr = convertInToOr(bb, leftKeys, valueList, isNotIn);
- return;
- }
-
- // Otherwise, let convertExists translate
- // values list into an inline table for the
- // reference to Q below.
- }
-
- // Project out the search columns from the left side
-
- // Q1:
- // "select from emp where emp.deptno in (select col1 from T)"
- //
- // is converted to
- //
- // "select from
- // emp inner join (select distinct col1 from T)) q
- // on emp.deptno = q.col1
- //
- // Q2:
- // "select from emp where emp.deptno not in (Q)"
- //
- // is converted to
- //
- // "select from
- // emp left outer join (select distinct col1, TRUE from T) q
- // on emp.deptno = q.col1
- // where emp.deptno <> null
- // and q.indicator <> TRUE"
- //
- final boolean outerJoin = bb.subqueryNeedsOuterJoin || isNotIn || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
- final RelDataType targetRowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(leftKeyNode), null);
- converted = convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, outerJoin, targetRowType);
- if (converted.right) {
- // Generate
- // emp CROSS JOIN (SELECT COUNT(*) AS c,
- // COUNT(deptno) AS ck FROM dept)
- final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RelNode seek = converted.left.getInput(0); // fragile
- final int keyCount = leftKeys.size();
- final List<Integer> args = ImmutableIntList.range(0, keyCount);
- LogicalAggregate aggregate = LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null, ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT, false, ImmutableList.<Integer> of(), -1, longType, null), AggregateCall.create(SqlStdOperatorTable.COUNT, false, args, -1, longType, null)));
- LogicalJoin join = LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), ImmutableSet.<CorrelationId> of(), JoinRelType.INNER);
- bb.setRoot(join, false);
- }
- RexNode rex = bb.register(converted.left, outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
-
- subQuery.expr = translateIn(subQuery, bb.root, rex);
- if (isNotIn) {
- subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ /**
+ * Push down all the NOT logical operators into any IN/NOT IN operators.
+ *
+ * @param sqlNode the root node from which to look for NOT operators
+ * @return the transformed SqlNode representation with NOT pushed down.
+ */
+ private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+ SqlCall sqlCall = (SqlCall) sqlNode;
+ if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
+ || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+ for (int i = 0; i < sqlOperands.length; i++) {
+ sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ }
+ return sqlNode;
+ } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode childNode = sqlCall.operand(0);
+ assert childNode instanceof SqlCall;
+ SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+ if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+ SqlNode[] andOperands = childSqlCall.getOperands();
+ SqlNode[] orOperands = new SqlNode[andOperands.length];
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ andOperands[i]);
+ }
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = pushDownNotForIn(orOperands[i]);
+ }
+ return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
+ orOperands[0], orOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+ SqlNode[] orOperands = childSqlCall.getOperands();
+ SqlNode[] andOperands = new SqlNode[orOperands.length];
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ orOperands[i]);
+ }
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = pushDownNotForIn(andOperands[i]);
+ }
+ return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
+ andOperands[0], andOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode[] notOperands = childSqlCall.getOperands();
+ assert notOperands.length == 1;
+ return pushDownNotForIn(notOperands[0]);
+ } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+ SqlNode[] inOperands = childSqlCall.getOperands();
+ SqlInOperator inOp =
+ (SqlInOperator) childSqlCall.getOperator();
+ if (inOp.isNotIn()) {
+ return SqlStdOperatorTable.IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ } else {
+ return SqlStdOperatorTable.NOT_IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ }
+ } else {
+ // childSqlCall is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // sqlNode is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
}
+ } else {
+ // tree rooted at sqlNode does not contain inOperator
+ return sqlNode;
+ }
+ }
+
+ /**
+ * Converts a WHERE clause.
+ *
+ * @param bb Blackboard
+ * @param where WHERE clause, may be null
+ */
+ private void convertWhere(
+ final Blackboard bb,
+ final SqlNode where) {
+ if (where == null) {
return;
+ }
+ SqlNode newWhere = pushDownNotForIn(where);
+ replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ final RexNode convertedWhere = bb.convertExpression(newWhere);
- case EXISTS:
- // "select from emp where exists (select a from T)"
- //
- // is converted to the following if the subquery is correlated:
- //
- // "select from emp left outer join (select AGG_TRUE() as indicator
- // from T group by corr_var) q where q.indicator is true"
- //
- // If there is no correlation, the expression is replaced with a
- // boolean indicating whether the subquery returned 0 or >= 1 row.
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(0);
- if (!config.isExpand()) {
+ // only allocate filter if the condition is not TRUE
+ if (convertedWhere.isAlwaysTrue()) {
+ return;
+ }
+
+ final RelFactories.FilterFactory factory =
+ RelFactories.DEFAULT_FILTER_FACTORY;
+ final RelNode filter = factory.createFilter(bb.root, convertedWhere);
+ final RelNode r;
+ final CorrelationUse p = getCorrelationUse(bb, filter);
+ if (p != null) {
+ assert p.r instanceof Filter;
+ Filter f = (Filter) p.r;
+ r = LogicalFilter.create(f.getInput(), f.getCondition(),
+ ImmutableSet.of(p.id));
+ } else {
+ r = filter;
+ }
+
+ bb.setRoot(r, false);
+ }
+
+ private void replaceSubQueries(
+ final Blackboard bb,
+ final SqlNode expr,
+ RelOptUtil.Logic logic) {
+ findSubQueries(bb, expr, logic, false);
+ for (SubQuery node : bb.subQueryList) {
+ substituteSubQuery(bb, node);
+ }
+ }
+
+ private void substituteSubQuery(Blackboard bb, SubQuery subQuery) {
+ final RexNode expr = subQuery.expr;
+ if (expr != null) {
+ // Already done.
+ return;
+ }
+
+ final SqlBasicCall call;
+ final RelNode rel;
+ final SqlNode query;
+ final RelOptUtil.Exists converted;
+ switch (subQuery.node.getKind()) {
+ case CURSOR:
+ convertCursor(bb, subQuery);
return;
- }
- converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, subQuery.logic, true, null);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
+
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ case ARRAY_QUERY_CONSTRUCTOR:
+ rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
+ subQuery.expr = bb.register(rel, JoinRelType.INNER);
return;
- }
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
- case SCALAR_QUERY:
- // Convert the subquery. If it's non-correlated, convert it
- // to a constant expression.
- if (!config.isExpand()) {
+ case IN:
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(1);
+ if (!config.isExpand() && !(query instanceof SqlNodeList)) {
+ return;
+ }
+ final SqlNode leftKeyNode = call.operand(0);
+
+ final List<RexNode> leftKeys;
+ switch (leftKeyNode.getKind()) {
+ case ROW:
+ leftKeys = Lists.newArrayList();
+ for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
+ leftKeys.add(bb.convertExpression(sqlExpr));
+ }
+ break;
+ default:
+ leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
+ }
+
+ final boolean notIn = ((SqlInOperator) call.getOperator()).isNotIn();
+ if (query instanceof SqlNodeList) {
+ SqlNodeList valueList = (SqlNodeList) query;
+ if (!containsNullLiteral(valueList)
+ && valueList.size() < config.getInSubQueryThreshold()) {
+ // We're under the threshold, so convert to OR.
+ subQuery.expr =
+ convertInToOr(
+ bb,
+ leftKeys,
+ valueList,
+ notIn);
+ return;
+ }
+
+ // Otherwise, let convertExists translate
+ // values list into an inline table for the
+ // reference to Q below.
+ }
+
+ // Project out the search columns from the left side
+
+ // Q1:
+ // "select from emp where emp.deptno in (select col1 from T)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp inner join (select distinct col1 from T)) q
+ // on emp.deptno = q.col1
+ //
+ // Q2:
+ // "select from emp where emp.deptno not in (Q)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp left outer join (select distinct col1, TRUE from T) q
+ // on emp.deptno = q.col1
+ // where emp.deptno <> null
+ // and q.indicator <> TRUE"
+ //
+ final RelDataType targetRowType =
+ SqlTypeUtil.promoteToRowType(typeFactory,
+ validator.getValidatedNodeType(leftKeyNode), null);
+ converted =
+ convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic,
+ notIn, targetRowType);
+ if (converted.indicator) {
+ // Generate
+ // emp CROSS JOIN (SELECT COUNT(*) AS c,
+ // COUNT(deptno) AS ck FROM dept)
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RelNode seek = converted.r.getInput(0); // fragile
+ final int keyCount = leftKeys.size();
+ final List<Integer> args = ImmutableIntList.range(0, keyCount);
+ LogicalAggregate aggregate =
+ LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
+ ImmutableList.of(
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ ImmutableList.<Integer>of(), -1, longType, null),
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ args, -1, longType, null)));
+ LogicalJoin join =
+ LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
+ ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
+ bb.setRoot(join, false);
+ }
+ final RexNode rex =
+ bb.register(converted.r,
+ converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
+ leftKeys);
+
+ RelOptUtil.Logic logic = subQuery.logic;
+ switch (logic) {
+ case TRUE_FALSE_UNKNOWN:
+ case UNKNOWN_AS_TRUE:
+ if (!converted.indicator) {
+ logic = RelOptUtil.Logic.TRUE_FALSE;
+ }
+ }
+ subQuery.expr = translateIn(logic, bb.root, rex);
+ if (notIn) {
+ subQuery.expr =
+ rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ }
return;
- }
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(0);
- converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true, null);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
+
+ case EXISTS:
+ // "select from emp where exists (select a from T)"
+ //
+ // is converted to the following if the sub-query is correlated:
+ //
+ // "select from emp left outer join (select AGG_TRUE() as indicator
+ // from T group by corr_var) q where q.indicator is true"
+ //
+ // If there is no correlation, the expression is replaced with a
+ // boolean indicating whether the sub-query returned 0 or >= 1 row.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(0);
+ if (!config.isExpand()) {
+ return;
+ }
+ converted = convertExists(query, RelOptUtil.SubQueryType.EXISTS,
+ subQuery.logic, true, null);
+ assert !converted.indicator;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
+ return;
+ }
+ subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
return;
- }
- rel = convertToSingleValueSubq(query, converted.left);
- subQuery.expr = bb.register(rel, JoinRelType.LEFT);
- return;
- case SELECT:
- // This is used when converting multiset queries:
- //
- // select * from unnest(select multiset[deptno] from emps);
- //
- converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true, null);
- assert !converted.right;
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
+ case SCALAR_QUERY:
+ // Convert the sub-query. If it's non-correlated, convert it
+ // to a constant expression.
+ if (!config.isExpand()) {
+ return;
+ }
+ call = (SqlBasicCall) subQuery.node;
+ query = call.operand(0);
+ converted = convertExists(query, RelOptUtil.SubQueryType.SCALAR,
+ subQuery.logic, true, null);
+ assert !converted.indicator;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
+ return;
+ }
+ rel = convertToSingleValueSubq(query, converted.r);
+ subQuery.expr = bb.register(rel, JoinRelType.LEFT);
+ return;
+
+ case SELECT:
+ // This is used when converting multiset queries:
+ //
+ // select * from unnest(select multiset[deptno] from emps);
+ //
+ converted = convertExists(subQuery.node, RelOptUtil.SubQueryType.SCALAR,
+ subQuery.logic, true, null);
+ assert !converted.indicator;
+ subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
+ return;
- default:
- throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
+ default:
+ throw Util.newInternal("unexpected kind of sub-query :" + subQuery.node);
}
}
- private RexNode translateIn(SubQuery subQuery, RelNode root, final RexNode rex) {
- switch (subQuery.logic) {
- case TRUE:
- return rexBuilder.makeLiteral(true);
+ private RexNode translateIn(RelOptUtil.Logic logic, RelNode root,
+ final RexNode rex) {
+ switch (logic) {
+ case TRUE:
+ return rexBuilder.makeLiteral(true);
+
+ case TRUE_FALSE:
+ case UNKNOWN_AS_FALSE:
+ assert rex instanceof RexRangeRef;
+ final int fieldCount = rex.getType().getFieldCount();
+ RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
+
+ // Then append the IS NOT NULL(leftKeysForIn).
+ //
+ // RexRangeRef contains the following fields:
+ // leftKeysForIn,
+ // rightKeysForIn (the original sub-query select list),
+ // nullIndicator
+ //
+ // The first two lists contain the same number of fields.
+ final int k = (fieldCount - 1) / 2;
+ for (int i = 0; i < k; i++) {
+ rexNode =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.AND,
+ rexNode,
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.IS_NOT_NULL,
+ rexBuilder.makeFieldAccess(rex, i)));
+ }
+ return rexNode;
+
+ case TRUE_FALSE_UNKNOWN:
+ case UNKNOWN_AS_TRUE:
+ // select e.deptno,
+ // case
+ // when ct.c = 0 then false
+ // when dt.i is not null then true
+ // when e.deptno is null then null
+ // when ct.ck < ct.c then null
+ // else false
+ // end
+ // from e
+ // cross join (select count(*) as c, count(deptno) as ck from v) as ct
+ // left join (select distinct deptno, true as i from v) as dt
+ // on e.deptno = dt.deptno
+ final Join join = (Join) root;
+ final Project left = (Project) join.getLeft();
+ final RelNode leftLeft = ((Join) left.getInput()).getLeft();
+ final int leftLeftCount = leftLeft.getRowType().getFieldCount();
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
+ final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
+ final RexNode iRef =
+ rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+
+ final RexLiteral zero =
+ rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
+ final RexNode unknownLiteral =
+ rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
+
+ final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
+ falseLiteral,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
+ trueLiteral);
+ final JoinInfo joinInfo = join.analyzeCondition();
+ for (int leftKey : joinInfo.leftKeys) {
+ final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
+ unknownLiteral);
+ }
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
+ unknownLiteral,
+ falseLiteral);
- case UNKNOWN_AS_FALSE:
- assert rex instanceof RexRangeRef;
- final int fieldCount = rex.getType().getFieldCount();
- RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
- rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
-
- // Then append the IS NOT NULL(leftKeysForIn).
- //
- // RexRangeRef contains the following fields:
- // leftKeysForIn,
- // rightKeysForIn (the original subquery select list),
- // nullIndicator
- //
- // The first two lists contain the same number of fields.
- final int k = (fieldCount - 1) / 2;
- for (int i = 0; i < k; i++) {
- rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, rexNode, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeFieldAccess(rex, i)));
- }
- return rexNode;
-
- case TRUE_FALSE_UNKNOWN:
- case UNKNOWN_AS_TRUE:
- // select e.deptno,
- // case
- // when ct.c = 0 then false
- // when dt.i is not null then true
- // when e.deptno is null then null
- // when ct.ck < ct.c then null
- // else false
- // end
- // from e
- // cross join (select count(*) as c, count(deptno) as ck from v) as ct
- // left join (select distinct deptno, true as i from v) as dt
- // on e.deptno = dt.deptno
- final Join join = (Join) root;
- final Project left = (Project) join.getLeft();
- final RelNode leftLeft = ((Join) left.getInput()).getLeft();
- final int leftLeftCount = leftLeft.getRowType().getFieldCount();
- final RelDataType nullableBooleanType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
- final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
- final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
- final RexNode iRef = rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
-
- final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
- final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
- final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
- final RexNode unknownLiteral = rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
-
- final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), falseLiteral, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), trueLiteral);
- final JoinInfo joinInfo = join.analyzeCondition();
- for (int leftKey : joinInfo.leftKeys) {
- final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral);
- }
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), unknownLiteral, falseLiteral);
+ return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
- return rexBuilder.makeCall(nullableBooleanType, SqlStdOperatorTable.CASE, args.build());
+ default:
+ throw new AssertionError(logic);
+ }
+ }
- default:
- throw new AssertionError(subQuery.logic);
+ private static boolean containsNullLiteral(SqlNodeList valueList) {
+ for (SqlNode node : valueList.getList()) {
+ if (node instanceof SqlLiteral) {
+ SqlLiteral lit = (SqlLiteral) node;
+ if (lit.getValue() == null) {
+ return true;
+ }
+ }
}
+ return false;
}
/**
- * Determines if a subquery is non-correlated and if so, converts it to a
+ * Determines if a sub-query is non-correlated and if so, converts it to a
* constant.
*
- * @param subQuery the call that references the subquery
- * @param bb blackboard used to convert the subquery
- * @param converted RelNode tree corresponding to the subquery
- * @param isExists true if the subquery is part of an EXISTS expression
- * @return if the subquery can be converted to a constant
+ * @param subQuery the call that references the sub-query
+ * @param bb blackboard used to convert the sub-query
+ * @param converted RelNode tree corresponding to the sub-query
+ * @param isExists true if the sub-query is part of an EXISTS expression
+ * @return Whether the sub-query can be converted to a constant
*/
- private boolean convertNonCorrelatedSubQuery(SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists) {
+ private boolean convertNonCorrelatedSubQuery(
+ SubQuery subQuery,
+ Blackboard bb,
+ RelNode converted,
+ boolean isExists) {
SqlCall call = (SqlBasicCall) subQuery.node;
- if (subqueryConverter.canConvertSubquery() && isSubQueryNonCorrelated(converted, bb)) {
- // First check if the subquery has already been converted
- // because it's a nested subquery. If so, don't re-evaluate
+ if (subQueryConverter.canConvertSubQuery()
+ && isSubQueryNonCorrelated(converted, bb)) {
+ // First check if the sub-query has already been converted
+ // because it's a nested sub-query. If so, don't re-evaluate
// it again.
RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
if (constExpr == null) {
- constExpr = subqueryConverter.convertSubquery(call, this, isExists, config.isExplain());
+ constExpr =
+ subQueryConverter.convertSubQuery(
+ call,
+ this,
+ isExists,
+ config.isExplain());
}
if (constExpr != null) {
subQuery.expr = constExpr;
@@ -1213,14 +1399,17 @@ public class SqlToRelConverter {
* @param plan the original RelNode tree corresponding to the statement
* @return the converted RelNode tree
*/
- public RelNode convertToSingleValueSubq(SqlNode query, RelNode plan) {
+ public RelNode convertToSingleValueSubq(
+ SqlNode query,
+ RelNode plan) {
// Check whether query is guaranteed to produce a single value.
if (query instanceof SqlSelect) {
SqlSelect select = (SqlSelect) query;
SqlNodeList selectList = select.getSelectList();
SqlNodeList groupList = select.getGroup();
- if ((selectList.size() == 1) && ((groupList == null) || (groupList.size() == 0))) {
+ if ((selectList.size() == 1)
+ && ((groupList == null) || (groupList.size() == 0))) {
SqlNode selectExpr = selectList.get(0);
if (selectExpr instanceof SqlCall) {
SqlCall selectExprCall = (SqlCall) selectExpr;
@@ -1231,7 +1420,8 @@ public class SqlToRelConverter {
// If there is a limit with 0 or 1,
// it is ensured to produce a single value
- if (select.getFetch() != null && select.getFetch() instanceof SqlNumericLiteral) {
+ if (select.getFetch() != null
+ && select.getFetch() instanceof SqlNumericLiteral) {
SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
return plan;
@@ -1243,13 +1433,17 @@ public class SqlToRelConverter {
// it is necessary to look into the operands to determine
// whether SingleValueAgg is necessary
SqlCall exprCall = (SqlCall) query;
- if (exprCall.getOperator() instanceof SqlValuesOperator && Util.isSingleValue(exprCall)) {
+ if (exprCall.getOperator()
+ instanceof SqlValuesOperator
+ && Util.isSingleValue(exprCall)) {
return plan;
}
}
// If not, project SingleValueAgg
- return RelOptUtil.createSingleValueAggRel(cluster, plan);
+ return RelOptUtil.createSingleValueAggRel(
+ cluster,
+ plan);
}
/**
@@ -1260,30 +1454,52 @@ public class SqlToRelConverter {
* @param isNotIn is this a NOT IN operator
* @return converted expression
*/
- private RexNode convertInToOr(final Blackboard bb, final List<RexNode> leftKeys, SqlNodeList valuesList, boolean isNotIn) {
+ private RexNode convertInToOr(
+ final Blackboard bb,
+ final List<RexNode> leftKeys,
+ SqlNodeList valuesList,
+ boolean isNotIn) {
final List<RexNode> comparisons = new ArrayList<>();
for (SqlNode rightVals : valuesList) {
RexNode rexComparison;
if (leftKeys.size() == 1) {
- rexComparison = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, leftKeys.get(0), ensureSqlType(leftKeys.get(0).getType(), bb.convertExpression(rightVals)));
+ rexComparison =
+ rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ leftKeys.get(0),
+ ensureSqlType(leftKeys.get(0).getType(),
+ bb.convertExpression(rightVals)));
} else {
assert rightVals instanceof SqlCall;
final SqlBasicCall call = (SqlBasicCall) rightVals;
- assert (call.getOperator() instanceof SqlRowOperator) && call.operandCount() == leftKeys.size();
- rexComparison = RexUtil.composeConjunction(rexBuilder, Iterables.transform(Pair.zip(leftKeys, call.getOperandList()), new Function<Pair<RexNode, SqlNode>, RexNode>() {
- public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, pair.left, ensureSqlType(pair.left.getType(), bb.convertExpression(pair.right)));
- }
- }), false);
+ assert (call.getOperator() instanceof SqlRowOperator)
+ && call.operandCount() == leftKeys.size();
+ rexComparison =
+ RexUtil.composeConjunction(
+ rexBuilder,
+ Iterables.transform(
+ Pair.zip(leftKeys, call.getOperandList()),
+ new Function<Pair<RexNode, SqlNode>, RexNode>() {
+ public RexNode apply(Pair<RexNode, SqlNode> pair) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ pair.left,
+ ensureSqlType(pair.left.getType(),
+ bb.convertExpression(pair.right)));
+ }
+ }),
+ false);
}
comparisons.add(rexComparison);
}
- RexNode result = RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+ RexNode result =
+ RexUtil.composeDisjunction(rexBuilder, comparisons, true);
assert result != null;
if (isNotIn) {
- result = rexBuilder.makeCall(SqlStdOperatorTable.NOT, result);
+ result =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.NOT,
+ result);
}
return result;
@@ -1293,7 +1509,9 @@ public class SqlToRelConverter {
* cast if necessary. If the expression already has the right type family,
* returns the expression unchanged. */
private RexNode ensureSqlType(RelDataType type, RexNode node) {
- if (type.getSqlTypeName() == node.getType().getSqlTypeName() || (type.getSqlTypeName() == SqlTypeName.VARCHAR && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
+ if (type.getSqlTypeName() == node.getType().getSqlTypeName()
+ || (type.getSqlTypeName() == SqlTypeName.VARCHAR
+ && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
return node;
}
return rexBuilder.ensureType(type, node, true);
@@ -1307,17 +1525,17 @@ public class SqlToRelConverter {
* predicate. A threshold of 0 forces usage of an inline table in all cases; a
* threshold of Integer.MAX_VALUE forces usage of OR in all cases
*
- * @return threshold, default {@link #DEFAULT_IN_SUBQUERY_THRESHOLD}
+ * @return threshold, default {@link #DEFAULT_IN_SUB_QUERY_THRESHOLD}
*/
@Deprecated // to be removed before 2.0
protected int getInSubqueryThreshold() {
- //return config.getInSubqueryThreshold();
- /* OVERRIDE POINT */
+ //return config.getInSubQueryThreshold();
+ /* OVERRIDE POINT */
return Integer.MAX_VALUE;
}
/**
- * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
+ * Converts an EXISTS or IN predicate into a join. For EXISTS, the sub-query
* produces an indicator variable, and the result is a relational expression
* which outer joins that indicator to the original query. After performing
* the outer join, the condition will be TRUE if the EXISTS condition holds,
@@ -1325,23 +1543,34 @@ public class SqlToRelConverter {
*
* @param seek A query, for example 'select * from emp' or
* 'values (1,2,3)' or '('Foo', 34)'.
- * @param subqueryType Whether sub-query is IN, EXISTS or scalar
+ * @param subQueryType Whether sub-query is IN, EXISTS or scalar
* @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
* FALSE, UNKNOWN) will be required, or whether we can accept an
* approximation (say representing UNKNOWN as FALSE)
- * @param needsOuterJoin Whether an outer join is needed
+ * @param notIn Whether the operation is NOT IN
* @return join expression
* @pre extraExpr == null || extraName != null
*/
- private Pair<RelNode, Boolean> convertExists(SqlNode seek, RelOptUtil.SubqueryType subqueryType, RelOptUtil.Logic logic, boolean needsOuterJoin, RelDataType targetDataType) {
- final SqlValidatorScope seekScope = (seek instanceof SqlSelect) ? validator.getSelectScope((SqlSelect) seek) : null;
+ private RelOptUtil.Exists convertExists(
+ SqlNode seek,
+ RelOptUtil.SubQueryType subQueryType,
+ RelOptUtil.Logic logic,
+ boolean notIn,
+ RelDataType targetDataType) {
+ final SqlValidatorScope seekScope =
+ (seek instanceof SqlSelect)
+ ? validator.getSelectScope((SqlSelect) seek)
+ : null;
final Blackboard seekBb = createBlackboard(seekScope, null, false);
RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
- return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, needsOuterJoin);
+ return RelOptUtil.createExistsPlan(seekRel, subQueryType, logic, notIn);
}
- private RelNode convertQueryOrInList(Blackboard bb, SqlNode seek, RelDataType targetRowType) {
+ private RelNode convertQueryOrInList(
+ Blackboard bb,
+ SqlNode seek,
+ RelDataType targetRowType) {
// NOTE: Once we start accepting single-row queries as row constructors,
// there will be an ambiguity here for a case like X IN ((SELECT Y FROM
// Z)). The SQL standard resolves the ambiguity by saying that a lone
@@ -1349,25 +1578,40 @@ public class SqlToRelConverter {
// expression. The semantic difference is that a table expression can
// return multiple rows.
if (seek instanceof SqlNodeList) {
- return convertRowValues(bb, seek, ((SqlNodeList) seek).getList(), false, targetRowType);
+ return convertRowValues(
+ bb,
+ seek,
+ ((SqlNodeList) seek).getList(),
+ false,
+ targetRowType);
} else {
return convertQueryRecursive(seek, false, null).project();
}
}
- private RelNode convertRowValues(Blackboard bb, SqlNode rowList, Collection<SqlNode> rows, boolean allowLiteralsOnly, RelDataType targetRowType) {
+ private RelNode convertRowValues(
+ Blackboard bb,
+ SqlNode rowList,
+ Collection<SqlNode> rows,
+ boolean allowLiteralsOnly,
+ RelDataType targetRowType) {
// NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
// literals into a single LogicalValues; this gives the optimizer a smaller
// input tree. For everything else (computed expressions, row
- // subqueries), we union each row in as a projection on top of a
+ // sub-queries), we union each row in as a projection on top of a
// LogicalOneRow.
- final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList = ImmutableList.builder();
+ final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
+ ImmutableList.builder();
final RelDataType rowType;
if (targetRowType != null) {
rowType = targetRowType;
} else {
- rowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(rowList), null);
+ rowType =
+ SqlTypeUtil.promoteToRowType(
+ typeFactory,
+ validator.getValidatedNodeType(rowList),
+ null);
}
final List<RelNode> unionInputs = new ArrayList<>();
@@ -1377,7 +1621,12 @@ public class SqlToRelConverter {
call = (SqlBasicCall) node;
ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
- RexLiteral rexLiteral = convertLiteralInValuesList(operand.e, bb, rowType, operand.i);
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ operand.e,
+ bb,
+ rowType,
+ operand.i);
if ((rexLiteral == null) && allowLiteralsOnly) {
return null;
}
@@ -1393,7 +1642,12 @@ public class SqlToRelConverter {
continue;
}
} else {
- RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0);
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ node,
+ bb,
+ rowType,
+ 0);
if ((rexLiteral != null) && config.isCreateValuesRel()) {
tupleList.add(ImmutableList.of(rexLiteral));
continue;
@@ -1404,11 +1658,15 @@ public class SqlToRelConverter {
}
// convert "1" to "row(1)"
- call = (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(SqlParserPos.ZERO, node);
+ call =
+ (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
+ SqlParserPos.ZERO,
+ node);
}
unionInputs.add(convertRowConstructor(bb, call));
}
- LogicalValues values = LogicalValues.create(cluster, rowType, tupleList.build());
+ LogicalValues values =
+ LogicalValues.create(cluster, rowType, tupleList.build());
RelNode resultRel;
if (unionInputs.isEmpty()) {
resultRel = values;
@@ -1422,7 +1680,11 @@ public class SqlToRelConverter {
return resultRel;
}
- private RexLiteral convertLiteralInValuesList(SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField) {
+ private RexLiteral convertLiteralInValuesList(
+ SqlNode sqlNode,
+ Blackboard bb,
+ RelDataType rowType,
+ int iField) {
if (!(sqlNode instanceof SqlLiteral)) {
return null;
}
@@ -1435,7 +1697,10 @@ public class SqlToRelConverter {
return null;
}
- RexNode literalExpr = exprConverter.convertLiteral(bb, (SqlLiteral) sqlNode);
+ RexNode literalExpr =
+ exprConverter.convertLiteral(
+ bb,
+ (SqlLiteral) sqlNode);
if (!(literalExpr instanceof RexLiteral)) {
assert literalExpr.isA(SqlKind.CAST);
@@ -1452,14 +1717,24 @@ public class SqlToRelConverter {
Comparable value = literal.getValue();
if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
- BigDecimal roundedValue = NumberUtil.rescaleBigDecimal((BigDecimal) value, type.getScale());
- return rexBuilder.makeExactLiteral(roundedValue, type);
+ BigDecimal roundedValue =
+ NumberUtil.rescaleBigDecimal(
+ (BigDecimal) value,
+ type.getScale());
+ return rexBuilder.makeExactLiteral(
+ roundedValue,
+ type);
}
- if ((value instanceof NlsString) && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+ if ((value instanceof NlsString)
+ && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
// pad fixed character type
NlsString unpadded = (NlsString) value;
- return rexBuilder.makeCharLiteral(new NlsString(Spaces.padRight(unpadded.getValue(), type.getPrecision()), unpadded.getCharsetName(), unpadded.getCollation()));
+ return rexBuilder.makeCharLiteral(
+ new NlsString(
+ Spaces.padRight(unpadded.getValue(), type.getPrecision()),
+ unpadded.getCharsetName(),
+ unpadded.getCollation()));
}
return literal;
}
@@ -1481,67 +1756,78 @@ public class SqlToRelConverter {
* @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
* FALSE, UNKNOWN) will be required, or whether we can accept
* an approximation (say representing UNKNOWN as FALSE)
- * @param registerOnlyScalarSubqueries if set to true and the parse tree
+ * @param registerOnlyScalarSubQueries if set to true and the parse tree
* corresponds to a variation of a select
* node, only register it if it's a scalar
- * subquery
+ * sub-query
*/
- private void findSubqueries(Blackboard bb, SqlNode node, RelOptUtil.Logic logic, boolean registerOnlyScalarSubqueries) {
+ private void findSubQueries(
+ Blackboard bb,
+ SqlNode node,
+ RelOptUtil.Logic logic,
+ boolean registerOnlyScalarSubQueries) {
final SqlKind kind = node.getKind();
switch (kind) {
- case EXISTS:
- case SELECT:
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case ARRAY_QUERY_CONSTRUCTOR:
- case CURSOR:
- case SCALAR_QUERY:
- if (!registerOnlyScalarSubqueries || (kind == SqlKind.SCALAR_QUERY)) {
- bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
- }
- return;
- case IN:
- if (((SqlCall) node).getOp
<TRUNCATED>
[37/47] kylin git commit: KYLIN-2242 fix the bug that couldn’t parse partition column max/min value error
Posted by bi...@apache.org.
KYLIN-2242 fix the bug that couldn\u2019t parse partition column max/min value error
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a86569a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a86569a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a86569a5
Branch: refs/heads/KYLIN-2361
Commit: a86569a5c0b49f5a607f642a70af7a826fda3edf
Parents: 571b791
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jan 24 15:59:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jan 24 15:59:38 2017 +0800
----------------------------------------------------------------------
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a86569a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 79fe657..2efd718 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -18,7 +18,9 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -79,6 +81,10 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+
+ if (partitionCol == null) {
+ return;
+ }
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
Path colDir = new Path(factColumnsInputPath, partitionCol.getIdentity());
FileSystem fs = HadoopUtil.getWorkingFileSystem();
@@ -88,15 +94,19 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
}
FSDataInputStream is = null;
- long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
+ BufferedReader bufferedReader = null;
+ InputStreamReader isr = null;
+ long minValue, maxValue;
try {
is = fs.open(outputFile);
- long min = is.readLong();
- long max = is.readLong();
- minValue = Math.min(min, minValue);
- maxValue = Math.max(max, maxValue);
+ isr = new InputStreamReader(is);
+ bufferedReader = new BufferedReader(isr);
+ minValue = Long.parseLong(bufferedReader.readLine());
+ maxValue = Long.parseLong(bufferedReader.readLine());
} finally {
IOUtils.closeQuietly(is);
+ IOUtils.closeQuietly(isr);
+ IOUtils.closeQuietly(bufferedReader);
}
logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);
segment.setDateRangeStart(minValue);
[34/47] kylin git commit: Add HDFSResourceStore to ResourceStore
Posted by bi...@apache.org.
Add HDFSResourceStore to ResourceStore
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f
Branch: refs/heads/KYLIN-2361
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <95...@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 5 +++++
.../kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 13 ++++---------
.../apache/kylin/storage/hdfs/ITLockManagerTest.java | 14 +++++++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 3 ++-
.../apache/kylin/storage/hdfs/HDFSResourceStore.java | 4 +++-
5 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
} catch (Throwable e) {
logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString());
}
+ try {
+ knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore", ResourceStore.class));
+ } catch (Throwable e) {
+ logger.warn("Failed to load HDFSResourceStore impl class: " + e.toString());
+ }
}
return knownImpl;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
package org.apache.kylin.storage.hdfs;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
-/**
- * Created by xiefan on 17-1-10.
- */
public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
this.cleanupTestMetadata();
}
-
@Test
public void testResourceStoreBasic() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- ResourceStore store = new HDFSResourceStore(config);
+ String oldUrl = kylinConfig.getMetadataUrl();
+ kylinConfig.setProperty("kylin.metadata.url", "kylin_default_instance@hdfs");
+ HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
ResourceStoreTest.testAStore(store);
+ kylinConfig.setProperty("kylin.metadata.url", oldUrl);
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
private static final int REPETITIONS = QTY * 10;
+ private static final Logger logger = LoggerFactory.getLogger(ITLockManagerTest.class);
+
@Before
public void setup() throws Exception {
this.createTestMetadata();
@@ -66,7 +70,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy);
zkClient.start();
manager = new LockManager(kylinConfig, lockRootPath);
- System.out.println("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
+ logger.info("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
}
@@ -75,7 +79,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
this.cleanupTestMetadata();
zkClient.delete().deletingChildrenIfNeeded().forPath(lockRootPath);
List<String> nodes = zkClient.getChildren().forPath("/");
- System.out.println("nodes in zk after delete : " + nodes);
+ logger.info("nodes in zk after delete : " + nodes);
manager.close();
}
@@ -85,7 +89,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
ResourceLock lock = manager.getLock("/dictionary/numberdict.json");
lock.acquire();
manager.releaseLock(lock);
- System.out.println(zkClient.getChildren().forPath(lockRootPath + "/dictionary"));
+ logger.info(zkClient.getChildren().forPath(lockRootPath + "/dictionary").toString());
List<String> nodes = zkClient.getChildren().forPath(lockRootPath + "/dictionary");
assertEquals(1, nodes.size());
assertEquals("numberdict.json", nodes.get(0));
@@ -193,10 +197,10 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
throw new IllegalStateException(clientName + " could not acquire the lock");
}
try {
- System.out.println(clientName + " has the lock");
+ logger.info(clientName + " has the lock");
resource.use();
} finally {
- System.out.println(clientName + " releasing the lock");
+ logger.info(clientName + " releasing the lock");
lock.release(); // always release the lock in a finally block
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 170e351..84670dd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -82,7 +82,8 @@ public class HBaseResourceStore extends ResourceStore {
int cut = metadataUrl.indexOf('@');
tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
+ if (!hbaseUrl.equals("hbase"))
+ throw new IOException("Can not create HBaseResourceStore. Url not match. Url:" + hbaseUrl);
createHTableIfNeeded(getAllInOneTableName());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index a746a97..bdd3917 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -57,12 +57,14 @@ public class HDFSResourceStore extends ResourceStore {
private LockManager lockManager;
- //public for test. Normal should be protected
public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
super(kylinConfig);
String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut);
+ String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+ if(!hdfsUrl.equals("hdfs"))
+ throw new IOException("Can not create HDFSResourceStore. Url not match. Url:" + hdfsUrl);
metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
logger.info("meta dir name :" + metaDirName);
createMetaFolder(metaDirName, kylinConfig);
[28/47] kylin git commit: minor, refine cdh profile name
Posted by bi...@apache.org.
minor, refine cdh profile name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db85d667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db85d667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db85d667
Branch: refs/heads/KYLIN-2361
Commit: db85d6679c5e1c067cfe2f1c58de0efb88044203
Parents: 87d1039
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Jan 23 16:14:33 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jan 23 16:14:33 2017 +0800
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/db85d667/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 520ac0c..bd1882e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1178,7 +1178,7 @@
</build>
</profile>
<profile>
- <id>cdh-5.7</id>
+ <id>cdh5.7</id>
<properties>
<hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
<yarn.version>2.6.0-cdh5.7.0</yarn.version>
[47/47] kylin git commit: KYLIN-2361 add Tomcat8 ordered class loader
Posted by bi...@apache.org.
KYLIN-2361 add Tomcat8 ordered class loader
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a3bd71c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a3bd71c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a3bd71c
Branch: refs/heads/KYLIN-2361
Commit: 9a3bd71c8e5ce9dc13e38560efc556dc862819a1
Parents: 2b60ac6
Author: Billy Liu <bi...@apache.org>
Authored: Sat Feb 4 11:41:49 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sat Feb 4 11:42:02 2017 +0800
----------------------------------------------------------------------
.../kylin/ext/CustomizedWebappClassloader.java | 4 +-
.../kylin/ext/OrderedWebResourceRoot.java | 286 +++++++++++++++++++
.../kylin/ext/WebappOrderedClassLoader.java | 66 +++++
3 files changed, 353 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index f241865..bbf4053 100644
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,12 @@
package org.apache.kylin.ext;
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
/**
* simple extension to standard ParallelWebappClassLoader
* the only difference is that CustomizedWebappClassloader is able to delegate more packages
* to parent classloaders
*/
-public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
+public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
/**
* Set of package names which are not allowed to be loaded from a webapp
* class loader without delegating first.
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
new file mode 100644
index 0000000..9784bd8
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.LifecycleListener;
+import org.apache.catalina.LifecycleState;
+import org.apache.catalina.TrackedWebResource;
+import org.apache.catalina.WebResource;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.WebResourceSet;
+
+public class OrderedWebResourceRoot implements WebResourceRoot {
+
+ private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
+
+ private static final Comparator<WebResource> WEB_RESOURCE_COMPARATOR = new Comparator<WebResource>() {
+ @Override
+ public int compare(WebResource o1, WebResource o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ };
+
+ private WebResourceRoot delegate;
+
+ public OrderedWebResourceRoot(WebResourceRoot delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public WebResource[] listResources(String path) {
+ WebResource[] webResources = delegate.listResources(path);
+
+ if (WEB_INF_LIB_PATH.equals(path)) {
+ Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
+ }
+
+ return webResources;
+ }
+
+ @Override
+ public void addLifecycleListener(LifecycleListener listener) {
+ delegate.addLifecycleListener(listener);
+ }
+
+ @Override
+ public LifecycleListener[] findLifecycleListeners() {
+ return delegate.findLifecycleListeners();
+ }
+
+ @Override
+ public void removeLifecycleListener(LifecycleListener listener) {
+ delegate.removeLifecycleListener(listener);
+ }
+
+ @Override
+ public void init() throws LifecycleException {
+ delegate.init();
+ }
+
+ @Override
+ public void start() throws LifecycleException {
+ delegate.start();
+ }
+
+ @Override
+ public void stop() throws LifecycleException {
+ delegate.stop();
+ }
+
+ @Override
+ public void destroy() throws LifecycleException {
+ delegate.destroy();
+ }
+
+ @Override
+ public LifecycleState getState() {
+ return delegate.getState();
+ }
+
+ @Override
+ public String getStateName() {
+ return delegate.getStateName();
+ }
+
+ @Override
+ public WebResource getResource(String path) {
+ return delegate.getResource(path);
+ }
+
+ @Override
+ public WebResource[] getResources(String path) {
+ return delegate.getResources(path);
+ }
+
+ @Override
+ public WebResource getClassLoaderResource(String path) {
+ return delegate.getClassLoaderResource(path);
+ }
+
+ @Override
+ public WebResource[] getClassLoaderResources(String path) {
+ return delegate.getClassLoaderResources(path);
+ }
+
+ @Override
+ public String[] list(String path) {
+ return delegate.list(path);
+ }
+
+ @Override
+ public Set<String> listWebAppPaths(String path) {
+ return delegate.listWebAppPaths(path);
+ }
+
+ @Override
+ public boolean mkdir(String path) {
+ return delegate.mkdir(path);
+ }
+
+ @Override
+ public boolean write(String path, InputStream is, boolean overwrite) {
+ return delegate.write(path, is, overwrite);
+ }
+
+ @Override
+ public void createWebResourceSet(ResourceSetType type, String webAppMount, URL url, String internalPath) {
+ delegate.createWebResourceSet(type, webAppMount, url, internalPath);
+ }
+
+ @Override
+ public void createWebResourceSet(ResourceSetType type, String webAppMount, String base, String archivePath,
+ String internalPath) {
+ delegate.createWebResourceSet(type, webAppMount, base, archivePath, internalPath);
+ }
+
+ @Override
+ public void addPreResources(WebResourceSet webResourceSet) {
+ delegate.addPreResources(webResourceSet);
+ }
+
+ @Override
+ public WebResourceSet[] getPreResources() {
+ return delegate.getPreResources();
+ }
+
+ @Override
+ public void addJarResources(WebResourceSet webResourceSet) {
+ delegate.addJarResources(webResourceSet);
+ }
+
+ @Override
+ public WebResourceSet[] getJarResources() {
+ return delegate.getJarResources();
+ }
+
+ @Override
+ public void addPostResources(WebResourceSet webResourceSet) {
+ delegate.addPostResources(webResourceSet);
+ }
+
+ @Override
+ public WebResourceSet[] getPostResources() {
+ return delegate.getPostResources();
+ }
+
+ @Override
+ public Context getContext() {
+ return delegate.getContext();
+ }
+
+ @Override
+ public void setContext(Context context) {
+ delegate.setContext(context);
+ }
+
+ @Override
+ public void setAllowLinking(boolean allowLinking) {
+ delegate.setAllowLinking(allowLinking);
+ }
+
+ @Override
+ public boolean getAllowLinking() {
+ return delegate.getAllowLinking();
+ }
+
+ @Override
+ public void setCachingAllowed(boolean cachingAllowed) {
+ delegate.setCachingAllowed(cachingAllowed);
+ }
+
+ @Override
+ public boolean isCachingAllowed() {
+ return delegate.isCachingAllowed();
+ }
+
+ @Override
+ public void setCacheTtl(long ttl) {
+ delegate.setCacheTtl(ttl);
+ }
+
+ @Override
+ public long getCacheTtl() {
+ return delegate.getCacheTtl();
+ }
+
+ @Override
+ public void setCacheMaxSize(long cacheMaxSize) {
+ delegate.setCacheMaxSize(cacheMaxSize);
+ }
+
+ @Override
+ public long getCacheMaxSize() {
+ return delegate.getCacheMaxSize();
+ }
+
+ @Override
+ public void setCacheObjectMaxSize(int cacheObjectMaxSize) {
+ delegate.setCacheObjectMaxSize(cacheObjectMaxSize);
+ }
+
+ @Override
+ public int getCacheObjectMaxSize() {
+ return delegate.getCacheObjectMaxSize();
+ }
+
+ @Override
+ public void setTrackLockedFiles(boolean trackLockedFiles) {
+ delegate.setTrackLockedFiles(trackLockedFiles);
+ }
+
+ @Override
+ public boolean getTrackLockedFiles() {
+ return delegate.getTrackLockedFiles();
+ }
+
+ @Override
+ public void backgroundProcess() {
+ delegate.backgroundProcess();
+ }
+
+ @Override
+ public void registerTrackedResource(TrackedWebResource trackedResource) {
+ delegate.registerTrackedResource(trackedResource);
+ }
+
+ @Override
+ public void deregisterTrackedResource(TrackedWebResource trackedResource) {
+ delegate.deregisterTrackedResource(trackedResource);
+ }
+
+ @Override
+ public List<URL> getBaseUrls() {
+ return delegate.getBaseUrls();
+ }
+
+ @Override
+ public void gc() {
+ delegate.gc();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
new file mode 100644
index 0000000..6a90e55
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.loader.ParallelWebappClassLoader;
+
+/**
+ * Modified from the openwide-java/tomcat-classloader-ordered in https://github.com/openwide-java/tomcat-classloader-ordered
+ *
+ * This classloader is designed to return the jar of WEB-INF lib in alphabetical order as it was the case with Tomcat
+ * 7.x.
+ *
+ * See the discussion in https://bz.apache.org/bugzilla/show_bug.cgi?id=57129 for more information.
+ */
+public class WebappOrderedClassLoader extends ParallelWebappClassLoader {
+
+ public WebappOrderedClassLoader() {
+ }
+
+ public WebappOrderedClassLoader(ClassLoader parent) {
+ super(parent);
+ }
+
+ @Override
+ public void setResources(WebResourceRoot resources) {
+ super.setResources(new OrderedWebResourceRoot(resources));
+ }
+
+ @Override
+ public WebappOrderedClassLoader copyWithoutTransformers() {
+ WebappOrderedClassLoader result = new WebappOrderedClassLoader(getParent());
+
+ super.copyStateWithoutTransformers(result);
+
+ try {
+ result.start();
+ } catch (LifecycleException e) {
+ throw new IllegalStateException(e);
+ }
+
+ return result;
+ }
+
+ @Override
+ protected Object getClassLoadingLock(String className) {
+ return this;
+ }
+}
\ No newline at end of file
[12/47] kylin git commit: KYLIN-2394 merge CALCITE-1592
Posted by bi...@apache.org.
KYLIN-2394 merge CALCITE-1592
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e
Branch: refs/heads/KYLIN-2361
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 20 15:42:37 2017 +0800
----------------------------------------------------------------------
.../calcite/sql2rel/SqlToRelConverter.java | 51 +++++++++-----------
1 file changed, 23 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
/**
* Push down all the NOT logical operators into any IN/NOT IN operators.
*
+ * @param scope Scope where {@code sqlNode} occurs
* @param sqlNode the root node from which to look for NOT operators
* @return the transformed SqlNode representation with NOT pushed down.
*/
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+ SqlNode sqlNode) {
if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
SqlCall sqlCall = (SqlCall) sqlNode;
if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
|| (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
}
- return sqlNode;
+ return reg(scope, sqlNode);
} else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
SqlNode childNode = sqlCall.operand(0);
assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
SqlNode[] andOperands = childSqlCall.getOperands();
SqlNode[] orOperands = new SqlNode[andOperands.length];
for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- andOperands[i]);
+ orOperands[i] = reg(scope, SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
}
for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
+ orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
}
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
- orOperands[0], orOperands[1]);
+ return reg(scope, SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]));
} else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
SqlNode[] orOperands = childSqlCall.getOperands();
SqlNode[] andOperands = new SqlNode[orOperands.length];
for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- orOperands[i]);
+ andOperands[i] = reg(scope, SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
}
for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
+ andOperands[i] = pushDownNotForIn(scope, andOperands[i]);
}
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- andOperands[0], andOperands[1]);
+ return reg(scope, SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]));
} else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
SqlNode[] notOperands = childSqlCall.getOperands();
assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
+ return pushDownNotForIn(scope, notOperands[0]);
} else if (childSqlCall.getOperator() instanceof SqlInOperator) {
SqlNode[] inOperands = childSqlCall.getOperands();
SqlInOperator inOp =
(SqlInOperator) childSqlCall.getOperator();
if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
+ return reg(scope, SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]));
} else {
- return SqlStdOperatorTable.NOT_IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
+ return reg(scope, SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]));
}
} else {
// childSqlCall is "leaf" node in a logical expression tree
@@ -1015,6 +1003,13 @@ public class SqlToRelConverter {
}
}
+ /** Registers with the validator a {@link SqlNode} that has been created
+ * during the Sql-to-Rel process. */
+ private static SqlNode reg(SqlValidatorScope scope, SqlNode e) {
+ scope.getValidator().deriveType(scope, e);
+ return e;
+ }
+
/**
* Converts a WHERE clause.
*
@@ -1027,7 +1022,7 @@ public class SqlToRelConverter {
if (where == null) {
return;
}
- SqlNode newWhere = pushDownNotForIn(where);
+ SqlNode newWhere = pushDownNotForIn(bb.scope, where);
replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
final RexNode convertedWhere = bb.convertExpression(newWhere);
@@ -2667,7 +2662,7 @@ public class SqlToRelConverter {
// Replace sub-queries in having here and modify having to use
// the replaced expressions
if (having != null) {
- SqlNode newHaving = pushDownNotForIn(having);
+ SqlNode newHaving = pushDownNotForIn(bb.scope, having);
replaceSubQueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
havingExpr = bb.convertExpression(newHaving);
if (havingExpr.isAlwaysTrue()) {
[14/47] kylin git commit: KYLIN-2308 Allow user to set more
columnFamily in web
Posted by bi...@apache.org.
KYLIN-2308 Allow user to set more columnFamily in web
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0
Branch: refs/heads/KYLIN-2361
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen <ka...@live.com>
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian <ji...@163.com>
Committed: Fri Jan 20 17:25:28 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++++++++++++++-
webapp/app/js/controllers/cubeEdit.js | 51 -----------
webapp/app/js/controllers/cubeSchema.js | 13 +++
webapp/app/js/filters/filter.js | 22 +++--
.../cubeDesigner/advanced_settings.html | 92 +++++++++++++++++++-
5 files changed, 194 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.isReuse=!$scope.isReuse;
}
- $scope.removeDictionaries = function(arr,element){
+ $scope.removeElement = function(arr,element){
var index = arr.indexOf(element);
if (index > -1) {
arr.splice(index, 1);
}
};
+ $scope.newColFamily = function (index) {
+ return {
+ "name": "F" + index,
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": []
+ }
+ ]
+ };
+ };
+
+ $scope.initColumnFamily = function () {
+ $scope.cubeMetaFrame.hbase_mapping.column_family = [];
+ var normalMeasures = [], distinctCountMeasures = [];
+ angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+ if (measure.function.expression === 'COUNT_DISTINCT') {
+ distinctCountMeasures.push(measure);
+ } else {
+ normalMeasures.push(measure);
+ }
+ });
+ if (normalMeasures.length > 0) {
+ var nmcf = $scope.newColFamily(1);
+ angular.forEach(normalMeasures, function (normalM, index) {
+ nmcf.columns[0].measure_refs.push(normalM.name);
+ });
+ $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+ }
+
+ if (distinctCountMeasures.length > 0) {
+ var dccf = $scope.newColFamily(2);
+ angular.forEach(distinctCountMeasures, function (dcm, index) {
+ dccf.columns[0].measure_refs.push(dcm.name);
+ });
+ $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+ }
+ };
+
+ $scope.getAllMeasureNames = function () {
+ var measures = [];
+ angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+ measures.push(measure.name);
+ });
+ return measures;
+ };
+
+ $scope.getAssignedMeasureNames = function () {
+ var assignedMeasures = [];
+ angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function (colFamily, index) {
+ angular.forEach(colFamily.columns[0].measure_refs, function (measure, index) {
+ assignedMeasures.push(measure);
+ });
+ });
+ return assignedMeasures;
+ };
+
+ if ($scope.getAllMeasureNames().length != $scope.getAssignedMeasureNames().length) {
+ $scope.initColumnFamily();
+ }
+
+
+ $scope.addColumnFamily = function () {
+ var colFamily = $scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 1);
+ $scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+ };
+
+ $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+ if (column_familys) {
+ column_familys[index] = colFamily;
+ }
+ };
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
return type;
};
- var ColFamily = function () {
- var index = 1;
- return function () {
- var newColFamily =
- {
- "name": "f" + index,
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": []
- }
- ]
- };
- index += 1;
-
- return newColFamily;
- }
- };
-
// ~ Define data
$scope.state = {
@@ -380,8 +361,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.prepareCube = function () {
- //generate column family
- generateColumnFamily();
//generate rowkey
reGenerateRowKey();
@@ -793,36 +772,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
return groups;
}
-
- // ~ private methods
- function generateColumnFamily() {
- $scope.cubeMetaFrame.hbase_mapping.column_family = [];
- var colFamily = ColFamily();
- var normalMeasures = [], distinctCountMeasures = [];
- angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
- if (measure.function.expression === 'COUNT_DISTINCT') {
- distinctCountMeasures.push(measure);
- } else {
- normalMeasures.push(measure);
- }
- });
- if (normalMeasures.length > 0) {
- var nmcf = colFamily();
- angular.forEach(normalMeasures, function (normalM, index) {
- nmcf.columns[0].measure_refs.push(normalM.name);
- });
- $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
- }
-
- if (distinctCountMeasures.length > 0) {
- var dccf = colFamily();
- angular.forEach(distinctCountMeasures, function (dcm, index) {
- dccf.columns[0].measure_refs.push(dcm.name);
- });
- $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
- }
- }
-
$scope.$watch('projectModel.selectedProject', function (newValue, oldValue) {
if(!$scope.projectModel.getSelectedProject()) {
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index af8ee7c..17371f2 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -275,6 +275,19 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
errors.push("At most one 'shard by' column is allowed.");
}
+ var cfMeasures = [];
+ angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family,function(cf){
+ angular.forEach(cf.columns[0].measure_refs, function (measure, index) {
+ cfMeasures.push(measure);
+ });
+ });
+
+ var uniqCfMeasures = _.uniq(cfMeasures);
+ if(uniqCfMeasures.length != $scope.cubeMetaFrame.measures.length) {
+ errors.push("All measures need to be assigned to column family");
+ }
+
+
var errorInfo = "";
angular.forEach(errors,function(item){
errorInfo+="\n"+item;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index aff4e3a..d097075 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -193,16 +193,26 @@ KylinApp
}).filter('inDimNotInMea', function ($filter) {
return function (inputArr, table, arr) {
var out=[];
- angular.forEach(arr,function(item){
- if(item.table==table){
- angular.forEach(inputArr,function(inputItem){
- if(item.columns.indexOf(inputItem.name)==-1){
+ angular.forEach(arr,function(item) {
+ if (item.table == table) {
+ angular.forEach(inputArr, function (inputItem) {
+ if (item.columns.indexOf(inputItem.name) == -1) {
out.push(inputItem);
}
});
}
});
+ }
+ }).filter('assignedMeasureNames', function ($filter) {
+ //return the measures that haven't assign to column family
+ return function (inputArr, assignedArr) {
+ var out = [];
+ angular.forEach(inputArr, function (inputItem) {
+ if (assignedArr.indexOf(inputItem) == -1) {
+ out.push(inputItem);
+ }
+ });
return out;
}
- })
- ;
+ });
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index f9e422c..bf95256 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -302,7 +302,7 @@
<i class="fa fa-pencil"></i>
</button>
<!--Remove Button -->
- <button class="btn btn-xs btn-danger" ng-click="removeDictionaries(cubeMetaFrame.dictionaries, dictionaries)" ng-disabled="instance.status=='READY'">
+ <button class="btn btn-xs btn-danger" ng-click="removeElement(cubeMetaFrame.dictionaries, dictionaries)" ng-disabled="instance.status=='READY'">
<i class="fa fa-trash-o"></i>
</button>
</td>
@@ -388,6 +388,89 @@
<button class="btn btn-link" ng-click="clearNewDictionaries()">Cancel</button>
</div>
</div>
+
+ <!--Edit ColumnFamily-->
+ <div class="form-group large-popover" style="overflow:auto">
+ <h3 style="margin-left:42px">Advanced ColumnFamily <i kylinpopover placement="right" title="Advanced ColumnFamily" template="AdvancedColumnFamilyTip.html" class="fa fa-info-circle"></i></h3>
+ <div style="margin-left:42px">
+ <div class="box-body">
+ <!-- VIEW MODE -->
+ <div class="row" ng-if="state.mode=='view'&& cubeMetaFrame.hbase_mapping.column_family.length > 0">
+ <table class="table table-striped table-hover">
+ <thead>
+ <tr>
+ <th class="col-xs-1">CF</th>
+ <th class="col-xs-11">Measures</th>
+ </tr>
+ </thead>
+ <tbody class="cube-dimension">
+ <tr ng-repeat="colFamily in cubeMetaFrame.hbase_mapping.column_family | filter:dimState.filter track by $index">
+ <!--ID -->
+ <td class="col-xs-1">
+ <b>{{colFamily.name}}</b>
+ </td>
+ <!--Name -->
+ <td class="col-xs-11">
+ <span>{{colFamily.columns[0].measure_refs}}</span>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
+ <!-- EDIT MODE -->
+ <div ng-if="state.mode=='edit'" class="form-group " style="width: 100%">
+ <table ng-if="cubeMetaFrame.hbase_mapping.column_family.length > 0"
+ class="table table-hover">
+
+ <tr class="row">
+ <th class="col-xs-1">CF</th>
+ <th class="col-xs-10">Measures</th>
+ <th class="col-xs-1">Actions</th>
+ </tr>
+
+ <tr ng-repeat="colFamily in cubeMetaFrame.hbase_mapping.column_family" ng-init="rowIndex = $index" class="row">
+ <td class="col-xs-1">
+ <b>{{colFamily.name}}</b>
+ </td>
+
+ <td class="col-xs-10">
+ <ui-select
+ ng-if="state.mode=='edit'"
+ style="width: 100%"
+ autofocus="true"
+ close-on-select="false"
+ on-select="refreshColumnFamily(cubeMetaFrame.hbase_mapping.column_family, rowIndex, colFamily)"
+ on-remove="refreshColumnFamily(cubeMetaFrame.hbase_mapping.column_family, rowIndex, colFamily)"
+ ng-model="colFamily.columns[0].measure_refs"
+ multiple>
+ <ui-select-match placeholder="Select Measure...">{{$item}}</ui-select-match>
+ <ui-select-choices repeat="measure in getAllMeasureNames() | filter:$select.search |assignedMeasureNames:getAssignedMeasureNames()">
+ {{measure}}
+ </ui-select-choices>
+ </ui-select>
+ </td>
+
+ <td class="col-xs-1">
+ <!--Remove Button -->
+ <button class="btn btn-xs btn-info" ng-click="removeElement(cubeMetaFrame.hbase_mapping.column_family, colFamily)">
+ <i class="fa fa-minus"></i>
+ </button>
+ </td>
+
+ </tr>
+ </table>
+ </div>
+
+ <div class="form-group" >
+ <button class="btn btn-sm btn-info" ng-click="addColumnFamily()" ng-show="state.mode=='edit'">
+ <i class="fa fa-plus"></i> ColumnFamily
+ </button>
+ </div>
+
+ </div>
+ </div>
+ </div>
</ng-form>
</div>
</ng-form>
@@ -419,3 +502,10 @@
<h4>Special settings for dictionaries. Leave blank by default.</h4>
</div>
</script>
+
+<script type="text/ng-template" id="AdvancedColumnFamilyTip.html">
+ <div>
+ <h4>If there are more than one ultrahigh cardinality precise count distinct measures,
+ you could assign these measures to more column family.</h4>
+ </div>
+</script>
[29/47] kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info
Posted by bi...@apache.org.
KYLIN-2217 use column\u2019s identity as path name and add more logging info
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e217ae84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e217ae84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e217ae84
Branch: refs/heads/KYLIN-2361
Commit: e217ae84e1bd163658176093de208fcdf3bdf43c
Parents: f6dda4f
Author: shaofengshi <sh...@apache.org>
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/util/HadoopUtil.java | 4 ++++
.../apache/kylin/cube/cli/DictionaryGeneratorCLI.java | 3 +++
.../org/apache/kylin/engine/mr/SortedColumnDFSFile.java | 4 ++++
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 2 +-
.../kylin/engine/mr/steps/CreateDictionaryJob.java | 9 +++++++--
.../engine/mr/steps/FactDistinctColumnsReducer.java | 11 +++++++----
.../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 ++--
.../hive/cardinality/HiveColumnCardinalityJob.java | 3 +++
8 files changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
}
public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final String filter) throws IOException {
+ if (fs.exists(baseDir) == false) {
+ return null;
+ }
+
FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
@Override
public boolean accept(Path path) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
if (dictProvider != null) {
Dictionary<String> dict = dictProvider.getDictionary(col);
if (dict != null) {
+ logger.debug("Dict for '" + col.getName() + "' has already been built, save it");
cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
} else {
+ logger.debug("Dict for '" + col.getName() + "' not pre-built, build it from " + inpTable.toString());
cubeMgr.buildDictionary(cubeSeg, col, inpTable);
}
} else {
+ logger.debug("Dict for '" + col.getName() + "' not pre-built, build it from " + inpTable.toString());
cubeMgr.buildDictionary(cubeSeg, col, inpTable);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d3f5cdc..f396b5a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -128,4 +128,8 @@ public class SortedColumnDFSFile implements ReadableTable {
return comparator;
}
+ @Override
+ public String toString() {
+ return dfsPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 567c1d0..44686d6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -75,7 +75,7 @@ import org.slf4j.LoggerFactory;
@SuppressWarnings("static-access")
public abstract class AbstractHadoopJob extends Configured implements Tool {
- protected static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
+ private static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job name. For example, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index e5d053b..be8c305 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -42,9 +42,13 @@ import org.apache.kylin.engine.mr.SortedColumnDFSFile;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CreateDictionaryJob extends AbstractHadoopJob {
+ private static final Logger logger = LoggerFactory.getLogger(CreateDictionaryJob.class);
+
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
@@ -68,11 +72,12 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path colDir = new Path(factColumnsInputPath, col.getName());
- FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+ Path colDir = new Path(factColumnsInputPath, col.getIdentity());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
Path dictFile = HadoopUtil.getFilterOnlyPath(fs, colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
if (dictFile == null) {
+ logger.info("Dict for '" + col.getName() + "' not pre-built.");
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5d2fb72..8c56bdf 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Preconditions;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
@@ -124,12 +125,14 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// normal col
col = columnList.get(reducerIdToColumnIndex.get(taskId));
+ Preconditions.checkNotNull(col);
+
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
isReducerLocalBuildDict = false;
}
- if (col != null && isReducerLocalBuildDict) {
+ if (isReducerLocalBuildDict) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
}
@@ -190,7 +193,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
} else {
byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
// output written to baseDir/colName/-r-00000 (etc)
- String fileName = col.getName() + "/";
+ String fileName = col.getIdentity() + "/";
mos.write(BatchConstants.CFG_OUTPUT_COLUMN, NullWritable.get(), new Text(keyBytes), fileName);
}
}
@@ -231,7 +234,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void outputPartitionInfo() throws IOException, InterruptedException {
if (col != null) {
// output written to baseDir/colName/colName.pci-r-00000 (etc)
- String partitionFileName = col.getName() + "/" + col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
+ String partitionFileName = col.getIdentity() + "/" + col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMinValue), partitionFileName);
mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMaxValue), partitionFileName);
@@ -241,7 +244,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
// output written to baseDir/colName/colName.rldict-r-00000 (etc)
- String dictFileName = col.getName() + "/" + col.getName() + DICT_FILE_POSTFIX;
+ String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) {
outputStream.writeUTF(dict.getClass().getName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 81d5c42..79fe657 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -80,8 +80,8 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
- FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+ Path colDir = new Path(factColumnsInputPath, partitionCol.getIdentity());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
Path outputFile = HadoopUtil.getFilterOnlyPath(fs, colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
if (outputFile == null) {
throw new IOException("fail to find the partition file in base dir: " + colDir);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index ea72b54..f439ccb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -37,6 +37,8 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This hadoop job will scan all rows of the hive table and then calculate the cardinality on each column.
@@ -44,6 +46,7 @@ import org.apache.kylin.metadata.model.TableDesc;
*
*/
public class HiveColumnCardinalityJob extends AbstractHadoopJob {
+ private static final Logger logger = LoggerFactory.getLogger(HiveColumnCardinalityJob.class);
public static final String JOB_TITLE = "Kylin Hive Column Cardinality Job";
@SuppressWarnings("static-access")
[45/47] kylin git commit: minor, stablize DefaultSchedulerTest
Posted by bi...@apache.org.
minor, stablize DefaultSchedulerTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a058bfb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a058bfb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a058bfb8
Branch: refs/heads/KYLIN-2361
Commit: a058bfb8b0490fe36b7fe4da026028411ed208a5
Parents: 546f88f
Author: Li Yang <li...@apache.org>
Authored: Fri Feb 3 13:43:56 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Feb 3 13:43:56 2017 +0800
----------------------------------------------------------------------
.../job/impl/threadpool/BaseSchedulerTest.java | 36 ++++++++++++++------
1 file changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a058bfb8/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index fdf5252..1ada9a1 100644
--- a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -31,11 +31,15 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.lock.MockJobLock;
import org.junit.After;
import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
+ private static final Logger logger = LoggerFactory.getLogger(BaseSchedulerTest.class);
+
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
@@ -70,19 +74,31 @@ public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
}
protected void waitForJobFinish(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- final ExecutableState status = job.getStatus();
- if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
+ int error = 0;
+ final int errorLimit = 3;
+
+ while (error < errorLimit) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ AbstractExecutable job = jobService.getJob(jobId);
+ ExecutableState status = job.getStatus();
+ if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
+ break;
}
+ } catch (Exception ex) {
+ logger.error("", ex);
+ error++;
}
}
+
+ if (error >= errorLimit) {
+ throw new RuntimeException("waitForJobFinish() encounters exceptions, see logs above");
+ }
}
protected void waitForJobStatus(String jobId, ExecutableState state, long interval) {
[19/47] kylin git commit: Merge branch 'KYLIN-2394'
Posted by bi...@apache.org.
Merge branch 'KYLIN-2394'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6
Branch: refs/heads/KYLIN-2361
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu <bi...@apache.org>
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sun Jan 22 17:43:16 2017 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 11 +-
.../calcite/sql2rel/SqlToRelConverter.java | 3782 +++++++++++-------
jdbc/pom.xml | 12 +-
.../java/org/apache/kylin/jdbc/KylinMeta.java | 2 +-
kylin-it/pom.xml | 14 +-
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 1 +
pom.xml | 10 +-
query/pom.xml | 10 +-
8 files changed, 2348 insertions(+), 1494 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
----------------------------------------------------------------------
[07/47] kylin git commit: KYLIN-2387 code refactor,
merge ImmutableBitmapCounter and MutableBitmapCounter into
RoaringBitmapCounter
Posted by bi...@apache.org.
KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter into RoaringBitmapCounter
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c3e7bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c3e7bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c3e7bf
Branch: refs/heads/KYLIN-2361
Commit: 38c3e7bf691ecdfd0f8d42fcc97065a0596be018
Parents: 61833d9
Author: gaodayue <ga...@meituan.com>
Authored: Thu Jan 19 15:23:34 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Jan 19 15:43:14 2017 +0800
----------------------------------------------------------------------
.../gridtable/AggregationCacheMemSizeTest.java | 11 +-
.../metadata/measure/MeasureCodecTest.java | 5 +-
.../kylin/measure/bitmap/BitmapAggregator.java | 25 +---
.../kylin/measure/bitmap/BitmapCounter.java | 28 +++-
.../measure/bitmap/BitmapCounterFactory.java | 30 ++++
.../BitmapIntersectDistinctCountAggFunc.java | 16 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 17 ++-
.../kylin/measure/bitmap/BitmapSerializer.java | 9 +-
.../measure/bitmap/ImmutableBitmapCounter.java | 108 --------------
.../measure/bitmap/MutableBitmapCounter.java | 60 --------
.../measure/bitmap/RoaringBitmapCounter.java | 147 +++++++++++++++++++
.../bitmap/RoaringBitmapCounterFactory.java | 47 ++++++
.../measure/AggregatorMemEstimateTest.java | 5 +-
.../measure/bitmap/BitmapAggregatorTest.java | 56 ++++++-
.../kylin/measure/bitmap/BitmapCounterTest.java | 32 +---
.../measure/bitmap/BitmapSerializerTest.java | 5 +-
16 files changed, 345 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 63c7672..f749fb4 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -33,7 +33,9 @@ import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
import org.apache.kylin.measure.basic.DoubleSumAggregator;
import org.apache.kylin.measure.basic.LongSumAggregator;
import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounterFactory;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.github.jamm.MemoryMeter;
@@ -43,13 +45,14 @@ import com.google.common.base.Stopwatch;
public class AggregationCacheMemSizeTest {
private static final MemoryMeter meter = new MemoryMeter();
- private static final MutableBitmapCounter[] bitmaps = new MutableBitmapCounter[5];
+ private static final BitmapCounterFactory bitmapFactory = RoaringBitmapCounterFactory.INSTANCE;
+ private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
private static final Random random = new Random();
// consider bitmaps with variant cardinality
static {
for (int i = 0; i < bitmaps.length; i++) {
- bitmaps[i] = new MutableBitmapCounter();
+ bitmaps[i] = bitmapFactory.newBitmap();
}
final int totalBits = 1_000_000;
@@ -116,7 +119,7 @@ public class AggregationCacheMemSizeTest {
}
private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
- MutableBitmapCounter counter = new MutableBitmapCounter();
+ BitmapCounter counter = bitmapFactory.newBitmap();
counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]);
BitmapAggregator result = new BitmapAggregator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 97c9751..7129a5e 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -25,7 +25,8 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -58,7 +59,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
HLLCounter hllc = new HLLCounter(16);
hllc.add("1234567");
hllc.add("abcdefg");
- MutableBitmapCounter bitmap = new MutableBitmapCounter();
+ BitmapCounter bitmap = RoaringBitmapCounterFactory.INSTANCE.newBitmap();
bitmap.add(123);
bitmap.add(45678);
bitmap.add(Integer.MAX_VALUE - 10);
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
index 2c91bfa..d57af48 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
@@ -21,47 +21,36 @@ package org.apache.kylin.measure.bitmap;
import org.apache.kylin.measure.MeasureAggregator;
public class BitmapAggregator extends MeasureAggregator<BitmapCounter> {
+ private static final BitmapCounterFactory bitmapFactory = RoaringBitmapCounterFactory.INSTANCE;
- private ImmutableBitmapCounter sum;
- private boolean isMutable;
+ private BitmapCounter sum;
@Override
public void reset() {
sum = null;
- isMutable = false;
}
@Override
public void aggregate(BitmapCounter value) {
- ImmutableBitmapCounter v = (ImmutableBitmapCounter) value;
-
// Here we optimize for case when group only has 1 value. In such situation, no
// aggregation is needed, so we just keep a reference to the first value, saving
// the cost of deserialization and merging.
if (sum == null) {
- sum = v;
+ sum = value;
return;
}
- MutableBitmapCounter mutable;
- if (!isMutable) { // when aggregate the second value
- mutable = sum.toMutable();
- sum = mutable;
- isMutable = true;
- } else { // for the third, forth, ...
- mutable = (MutableBitmapCounter) sum;
- }
- mutable.orWith(v);
+ sum.orWith(value);
}
@Override
public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) {
- MutableBitmapCounter merged = new MutableBitmapCounter();
+ BitmapCounter merged = bitmapFactory.newBitmap();
if (value1 != null) {
- merged.orWith((ImmutableBitmapCounter) value1);
+ merged.orWith(value1);
}
if (value2 != null) {
- merged.orWith((ImmutableBitmapCounter) value2);
+ merged.orWith(value2);
}
return merged;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index f07059c..0854b6d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -26,6 +26,30 @@ import java.util.Iterator;
* An implementation-agnostic bitmap type.
*/
public interface BitmapCounter extends Iterable<Integer> {
+
+ /**
+ * Add the value to the bitmap (set the value to "true"), whether it already appears or not.
+ * @param value integer value
+ */
+ void add(int value);
+
+ /**
+ * In-place bitwise OR (union) operation. The current bitmap is modified.
+ * @param another other bitmap
+ */
+ void orWith(BitmapCounter another);
+
+ /**
+ * In-place bitwise AND (intersection) operation. The current bitmap is modified.
+ * @param another other bitmap
+ */
+ void andWith(BitmapCounter another);
+
+ /**
+ * reset to an empty bitmap
+ */
+ void clear();
+
/**
* @return cardinality of the bitmap
*/
@@ -44,13 +68,13 @@ public interface BitmapCounter extends Iterable<Integer> {
/**
* Serialize this counter. The current counter is not modified.
*/
- void serialize(ByteBuffer out) throws IOException;
+ void write(ByteBuffer out) throws IOException;
/**
* Deserialize a counter from its serialized form.
* <p> After deserialize, any changes to `in` should not affect the returned counter.
*/
- BitmapCounter deserialize(ByteBuffer in) throws IOException;
+ void readFields(ByteBuffer in) throws IOException;
/**
* @return size of the counter stored in the current position of `in`.
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
new file mode 100644
index 0000000..da7748e
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounterFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.measure.bitmap;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface BitmapCounterFactory {
+ BitmapCounter newBitmap();
+
+ BitmapCounter newBitmap(int... values);
+
+ BitmapCounter newBitmap(ByteBuffer in) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
index dcdf945..cd4d306 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
@@ -28,9 +28,10 @@ import java.util.Map;
* requires an bitmap count distinct measure of uuid, and an dimension of event
*/
public class BitmapIntersectDistinctCountAggFunc {
+ private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
public static class RetentionPartialResult {
- Map<Object, MutableBitmapCounter> map;
+ Map<Object, BitmapCounter> map;
List keyList;
public RetentionPartialResult() {
@@ -42,12 +43,11 @@ public class BitmapIntersectDistinctCountAggFunc {
this.keyList = keyList;
}
if (this.keyList != null && this.keyList.contains(key)) {
- MutableBitmapCounter counter = map.get(key);
+ BitmapCounter counter = map.get(key);
if (counter == null) {
- counter = new MutableBitmapCounter();
- map.put(key, counter);
+ map.put(key, counter = factory.newBitmap());
}
- counter.orWith((ImmutableBitmapCounter) value);
+ counter.orWith((BitmapCounter) value);
}
}
@@ -61,11 +61,11 @@ public class BitmapIntersectDistinctCountAggFunc {
return 0;
}
}
- MutableBitmapCounter counter = null;
+ BitmapCounter counter = null;
for (Object key : keyList) {
- MutableBitmapCounter c = map.get(key);
+ BitmapCounter c = map.get(key);
if (counter == null) {
- counter = new MutableBitmapCounter();
+ counter = factory.newBitmap();
counter.orWith(c);
} else {
counter.andWith(c);
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index de2a34a..e4fb079 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -96,18 +96,19 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
@Override
public MeasureIngester<BitmapCounter> newIngester() {
+ final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
+
return new MeasureIngester<BitmapCounter>() {
- MutableBitmapCounter current = new MutableBitmapCounter();
+ BitmapCounter current = factory.newBitmap();
@Override
public BitmapCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
checkArgument(values.length == 1, "expect 1 value, got %s", Arrays.toString(values));
- MutableBitmapCounter bitmap = current;
- bitmap.clear();
+ current.clear();
if (values[0] == null) {
- return bitmap;
+ return current;
}
int id;
@@ -119,8 +120,8 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
id = Integer.parseInt(values[0]);
}
- bitmap.add(id);
- return bitmap;
+ current.add(id);
+ return current;
}
@Override
@@ -132,7 +133,7 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
Dictionary<String> sourceDict = oldDicts.get(colRef);
Dictionary<String> mergedDict = newDicts.get(colRef);
- MutableBitmapCounter retValue = new MutableBitmapCounter();
+ BitmapCounter retValue = factory.newBitmap();
for (int id : value) {
int newId;
String v = sourceDict.getValueFromId(id);
@@ -148,7 +149,7 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
@Override
public void reset() {
- current = new MutableBitmapCounter();
+ current = factory.newBitmap();
}
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
index 0e970de..c1b260d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
@@ -25,7 +25,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
- private static final BitmapCounter DELEGATE = new MutableBitmapCounter();
+ private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
+ private static final BitmapCounter DELEGATE = factory.newBitmap();
// called by reflection
public BitmapSerializer(DataType type) {
@@ -34,7 +35,7 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public void serialize(BitmapCounter value, ByteBuffer out) {
try {
- value.serialize(out);
+ value.write(out);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -42,9 +43,9 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public BitmapCounter deserialize(ByteBuffer in) {
-
try {
- return DELEGATE.deserialize(in);
+ return factory.newBitmap(in);
+
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/ImmutableBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/ImmutableBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/ImmutableBitmapCounter.java
deleted file mode 100644
index 753f089..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/ImmutableBitmapCounter.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.measure.bitmap;
-
-import org.apache.kylin.common.util.ByteBufferOutputStream;
-import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.BufferOverflowException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-/**
- * A thin wrapper around {@link ImmutableRoaringBitmap}.
- */
-public class ImmutableBitmapCounter implements BitmapCounter {
-
- protected ImmutableRoaringBitmap bitmap;
-
- public ImmutableBitmapCounter(ImmutableRoaringBitmap bitmap) {
- this.bitmap = bitmap;
- }
-
- @Override
- public long getCount() {
- return bitmap.getCardinality();
- }
-
- @Override
- public int getMemBytes() {
- return bitmap.getSizeInBytes();
- }
-
- @Override
- public Iterator<Integer> iterator() {
- return bitmap.iterator();
- }
-
- @Override
- public void serialize(ByteBuffer out) throws IOException {
- if (out.remaining() < bitmap.serializedSizeInBytes()) {
- throw new BufferOverflowException();
- }
- bitmap.serialize(new DataOutputStream(new ByteBufferOutputStream(out)));
- }
-
- @Override
- public BitmapCounter deserialize(ByteBuffer in) throws IOException {
- int size = peekLength(in);
- // make a copy of the content to be safe
- byte[] dst = new byte[size];
- in.get(dst);
-
- // just map the buffer, faster than deserialize
- ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(ByteBuffer.wrap(dst));
- return new ImmutableBitmapCounter(bitmap);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- // only look at the metadata of the bitmap, no deserialization happens
- ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(in);
- return bitmap.serializedSizeInBytes();
- }
-
- /**
- * Copies the content of this counter to a counter that can be modified.
- * @return a mutable counter
- */
- public MutableBitmapCounter toMutable() {
- MutableBitmapCounter mutable = new MutableBitmapCounter();
- mutable.orWith(this);
- return mutable;
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj instanceof ImmutableBitmapCounter) &&
- bitmap.equals(((ImmutableBitmapCounter) obj).bitmap);
- }
-
- @Override
- public int hashCode() {
- return bitmap.hashCode();
- }
-
- @Override
- public String toString() {
- return "BitmapCounter[" + getCount() + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/MutableBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/MutableBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/MutableBitmapCounter.java
deleted file mode 100644
index af01790..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/MutableBitmapCounter.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.measure.bitmap;
-
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-/**
- * A thin wrapper around {@link MutableRoaringBitmap}.
- */
-public class MutableBitmapCounter extends ImmutableBitmapCounter {
-
- public MutableBitmapCounter() {
- super(new MutableRoaringBitmap());
- }
-
- private MutableRoaringBitmap getBitmap() {
- return (MutableRoaringBitmap) bitmap;
- }
-
- public void clear() {
- getBitmap().clear();
- }
-
- public void add(int value) {
- getBitmap().add(value);
- }
-
- public void orWith(ImmutableBitmapCounter another) {
- getBitmap().or(another.bitmap);
- }
-
- public void andWith(ImmutableBitmapCounter another) {
- getBitmap().and(another.bitmap);
- }
-
- @Override
- public void serialize(ByteBuffer out) throws IOException {
- getBitmap().runOptimize();
- super.serialize(out);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
new file mode 100644
index 0000000..cd07d20
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.measure.bitmap;
+
+import org.apache.kylin.common.util.ByteBufferOutputStream;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+/**
+ * A {@link BitmapCounter} based on roaring bitmap.
+ */
+public class RoaringBitmapCounter implements BitmapCounter {
+
+ private ImmutableRoaringBitmap bitmap;
+
+ RoaringBitmapCounter() {
+ bitmap = new MutableRoaringBitmap();
+ }
+
+ RoaringBitmapCounter(ImmutableRoaringBitmap bitmap) {
+ this.bitmap = bitmap;
+ }
+
+ private MutableRoaringBitmap getMutableBitmap() {
+ if (bitmap instanceof MutableRoaringBitmap) {
+ return (MutableRoaringBitmap) bitmap;
+ }
+ // convert to mutable bitmap
+ MutableRoaringBitmap result = bitmap.toMutableRoaringBitmap();
+ bitmap = result;
+ return result;
+ }
+
+ @Override
+ public void add(int value) {
+ getMutableBitmap().add(value);
+ }
+
+ @Override
+ public void orWith(BitmapCounter another) {
+ if (another instanceof RoaringBitmapCounter) {
+ RoaringBitmapCounter input = (RoaringBitmapCounter) another;
+ getMutableBitmap().or(input.bitmap);
+ return;
+ }
+ throw new IllegalArgumentException("Unsupported type: " + another.getClass().getCanonicalName());
+ }
+
+ @Override
+ public void andWith(BitmapCounter another) {
+ if (another instanceof RoaringBitmapCounter) {
+ RoaringBitmapCounter input = (RoaringBitmapCounter) another;
+ getMutableBitmap().and(input.bitmap);
+ return;
+ }
+ throw new IllegalArgumentException("Unsupported type: " + another.getClass().getCanonicalName());
+ }
+
+ @Override
+ public void clear() {
+ bitmap = new MutableRoaringBitmap();
+ }
+
+ @Override
+ public long getCount() {
+ return bitmap.getCardinality();
+ }
+
+ @Override
+ public int getMemBytes() {
+ return bitmap.getSizeInBytes();
+ }
+
+ @Override
+ public Iterator<Integer> iterator() {
+ return bitmap.iterator();
+ }
+
+ @Override
+ public void write(ByteBuffer out) throws IOException {
+ if (bitmap instanceof MutableRoaringBitmap) {
+ getMutableBitmap().runOptimize();
+ }
+
+ if (out.remaining() < bitmap.serializedSizeInBytes()) {
+ throw new BufferOverflowException();
+ }
+ bitmap.serialize(new DataOutputStream(new ByteBufferOutputStream(out)));
+ }
+
+ @Override
+ public void readFields(ByteBuffer in) throws IOException {
+ int size = peekLength(in);
+ // make a copy of the content to be safe
+ byte[] dst = new byte[size];
+ in.get(dst);
+
+ // ImmutableRoaringBitmap only maps the buffer, thus faster than constructing a MutableRoaringBitmap.
+ // we'll convert to MutableRoaringBitmap later when mutate is needed
+ bitmap = new ImmutableRoaringBitmap(ByteBuffer.wrap(dst));
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ // only look at the metadata of the bitmap, no deserialization happens
+ ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(in);
+ return bitmap.serializedSizeInBytes();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return (obj instanceof RoaringBitmapCounter) &&
+ bitmap.equals(((RoaringBitmapCounter) obj).bitmap);
+ }
+
+ @Override
+ public int hashCode() {
+ return bitmap.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "RoaringBitmapCounter[" + getCount() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
new file mode 100644
index 0000000..a71df95
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.measure.bitmap;
+
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class RoaringBitmapCounterFactory implements BitmapCounterFactory {
+ public static final BitmapCounterFactory INSTANCE = new RoaringBitmapCounterFactory();
+
+ private RoaringBitmapCounterFactory() {}
+
+ @Override
+ public BitmapCounter newBitmap() {
+ return new RoaringBitmapCounter();
+ }
+
+ @Override
+ public BitmapCounter newBitmap(int... values) {
+ return new RoaringBitmapCounter(MutableRoaringBitmap.bitmapOf(values));
+ }
+
+ @Override
+ public BitmapCounter newBitmap(ByteBuffer in) throws IOException {
+ RoaringBitmapCounter counter = new RoaringBitmapCounter();
+ counter.readFields(in);
+ return counter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 39921c2..4e67d22 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -33,7 +33,8 @@ import org.apache.kylin.measure.basic.LongMaxAggregator;
import org.apache.kylin.measure.basic.LongMinAggregator;
import org.apache.kylin.measure.basic.LongSumAggregator;
import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
@@ -104,7 +105,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
hllcAggregator.aggregate(new HLLCounter(14));
BitmapAggregator bitmapAggregator = new BitmapAggregator();
- MutableBitmapCounter bitmapCounter = new MutableBitmapCounter();
+ BitmapCounter bitmapCounter = RoaringBitmapCounterFactory.INSTANCE.newBitmap();
for (int i = 4000; i <= 100000; i += 2) {
bitmapCounter.add(i);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
index a52e427..0b82fc4 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
@@ -19,30 +19,72 @@
package org.apache.kylin.measure.bitmap;
import org.junit.Test;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
public class BitmapAggregatorTest {
+ private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
@Test
public void testAggregator() {
BitmapAggregator aggregator = new BitmapAggregator();
assertNull(null, aggregator.getState());
- aggregator.aggregate(new ImmutableBitmapCounter(
- MutableRoaringBitmap.bitmapOf(10, 20, 30, 40)
- ));
+ aggregator.aggregate(factory.newBitmap(10, 20, 30, 40));
assertEquals(4, aggregator.getState().getCount());
- aggregator.aggregate(new ImmutableBitmapCounter(
- MutableRoaringBitmap.bitmapOf(25, 30, 35, 40, 45)
- ));
+ aggregator.aggregate(factory.newBitmap(25, 30, 35, 40, 45));
assertEquals(7, aggregator.getState().getCount());
aggregator.reset();
assertNull(aggregator.getState());
}
+ @Test
+ public void testAggregatorDeserializedCounter() throws IOException {
+ BitmapCounter counter1 = factory.newBitmap(1, 3, 5);
+ BitmapCounter counter2 = factory.newBitmap(1, 2, 4, 6);
+ BitmapCounter counter3 = factory.newBitmap(1, 5, 7);
+
+ ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
+ counter1.write(buffer);
+ counter2.write(buffer);
+ counter3.write(buffer);
+ buffer.flip();
+
+ BitmapAggregator aggregator = new BitmapAggregator();
+
+ // first
+ BitmapCounter next = factory.newBitmap(buffer);
+ assertEquals(counter1, next);
+
+ aggregator.aggregate(next);
+ assertEquals(counter1, aggregator.getState());
+
+ // second
+ next = factory.newBitmap(buffer);
+ assertEquals(counter2, next);
+
+ aggregator.aggregate(next);
+ assertEquals(6, aggregator.getState().getCount());
+
+ // third
+ next = factory.newBitmap(buffer);
+ assertEquals(counter3, next);
+
+ aggregator.aggregate(next);
+ assertEquals(7, aggregator.getState().getCount());
+
+ BitmapCounter result = factory.newBitmap();
+ result.orWith(counter1);
+ result.orWith(counter2);
+ result.orWith(counter3);
+ assertEquals(result, aggregator.getState());
+
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
index ee7733f..7194a23 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
@@ -19,23 +19,21 @@
package org.apache.kylin.measure.bitmap;
import org.junit.Test;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class BitmapCounterTest {
+ private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
@Test
public void testBitmapCounter() {
- ImmutableBitmapCounter counter = new ImmutableBitmapCounter(
- MutableRoaringBitmap.bitmapOf(10, 20, 30, 1000)
- );
+ BitmapCounter counter = factory.newBitmap(10, 20, 30, 1000);
assertEquals(4, counter.getCount());
assertTrue(counter.getMemBytes() > 0);
- MutableBitmapCounter counter2 = new MutableBitmapCounter();
+ BitmapCounter counter2 = factory.newBitmap();
assertEquals(0, counter2.getCount());
counter2.add(10);
counter2.add(30);
@@ -58,28 +56,4 @@ public class BitmapCounterTest {
assertEquals(0, counter2.getCount());
}
- @Test
- public void testToMutableBitmapCounter() {
- ImmutableBitmapCounter immutable = new ImmutableBitmapCounter(
- MutableRoaringBitmap.bitmapOf(10, 20, 30, 1000)
- );
- MutableBitmapCounter mutable = new MutableBitmapCounter();
- mutable.orWith(immutable);
-
- assertEquals(4, immutable.getCount());
- assertEquals(4, mutable.getCount());
- assertTrue(immutable.equals(mutable));
- assertTrue(mutable.equals(immutable));
-
- MutableBitmapCounter newCounter = immutable.toMutable();
- newCounter.add(40);
- assertEquals(4, immutable.getCount());
- assertEquals(5, newCounter.getCount());
-
- newCounter = mutable.toMutable();
- newCounter.add(40);
- assertEquals(4, mutable.getCount());
- assertEquals(5, newCounter.getCount());
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
index e5b828d..acbfe88 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
@@ -24,7 +24,6 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
@@ -47,9 +46,7 @@ public class BitmapSerializerTest extends LocalFileMetadataTestCase {
public void testBitmapSerDe() {
BitmapSerializer serializer = new BitmapSerializer(DataType.ANY);
- ImmutableBitmapCounter counter = new ImmutableBitmapCounter(
- MutableRoaringBitmap.bitmapOf(1, 1234, 5678, 100000)
- );
+ BitmapCounter counter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(1, 1234, 5678, 100000);
ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
serializer.serialize(counter, buffer);
[35/47] kylin git commit: KYLIN-2415 Change back default metadata
name to kylin_metadata
Posted by bi...@apache.org.
KYLIN-2415 Change back default metadata name to kylin_metadata
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d247c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d247c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d247c2
Branch: refs/heads/KYLIN-2361
Commit: b6d247c2c1412d7eb1024048ae7aad0ba5d1ea82
Parents: 971bf80
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jan 24 13:43:04 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jan 24 13:43:04 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../apache/kylin/metadata/realization/IRealizationConstants.java | 2 +-
examples/test_case_data/sandbox/kylin.properties | 2 +-
.../org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 2 +-
.../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java | 2 +-
.../org/apache/kylin/storage/hbase/util/StorageCleanupJob.java | 2 +-
.../main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java | 2 +-
tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 196a711..eceb886 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
### METADATA | ENV ###
# The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 75b38ff..07cb124 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,7 +216,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getMetadataUrlPrefix() {
String metadataUrl = getMetadataUrl();
- String defaultPrefix = "kylin_default_instance";
+ String defaultPrefix = "kylin_metadata";
if (metadataUrl.endsWith("@hbase")) {
int cut = metadataUrl.lastIndexOf('@');
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index adc53a1..2a3b01e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
/**
* For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_default_instance prefix) owns this htable
+ * which kylin server(represented by its kylin_metadata prefix) owns this htable
*/
public final static String HTableTag = "KYLIN_HOST";
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 2c2da91..fa0c6bc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
### STORAGE ###
# The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 47afb01..ff66048 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -43,7 +43,7 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
@Test
public void testResourceStoreBasic() throws Exception {
String oldUrl = kylinConfig.getMetadataUrl();
- kylinConfig.setProperty("kylin.metadata.url", "kylin_default_instance@hdfs");
+ kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hdfs");
HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
ResourceStoreTest.testAStore(store);
kylinConfig.setProperty("kylin.metadata.url", oldUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 84670dd..0901b54 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -59,7 +59,7 @@ public class HBaseResourceStore extends ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(HBaseResourceStore.class);
- private static final String DEFAULT_TABLE_NAME = "kylin_default_instance";
+ private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
private static final String FAMILY = "f";
private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
private static final String COLUMN = "c";
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 0784305..f6b65ab 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -190,7 +190,7 @@ public class StorageCleanupJob extends AbstractApplication {
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
// + "/kylin-.*");
- // TODO: when first use, /kylin/kylin_default_instance does not exist.
+ // TODO: when first use, /kylin/kylin_metadata does not exist.
FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
for (FileStatus status : fStatus) {
String path = status.getPath().getName();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index bdd3917..0ad6964 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -47,7 +47,7 @@ public class HDFSResourceStore extends ResourceStore {
private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 10;
- private static final String DEFAULT_FOLDER_NAME = "kylin_default_instance";
+ private static final String DEFAULT_FOLDER_NAME = "kylin_metadata";
private static final String DEFAULT_METADATA_FOLDER_NAME = "hdfs_metadata";
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index b48c076..477c58a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -190,7 +190,7 @@ public class StorageCleanupJob extends AbstractApplication {
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
// + "/kylin-.*");
- // TODO: when first use, /kylin/kylin_default_instance does not exist.
+ // TODO: when first use, /kylin/kylin_metadata does not exist.
FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
for (FileStatus status : fStatus) {
String path = status.getPath().getName();
[38/47] kylin git commit: KYLIN-2374 fix path bug in HDFSResourceStore
Posted by bi...@apache.org.
KYLIN-2374 fix path bug in HDFSResourceStore
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56293411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56293411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56293411
Branch: refs/heads/KYLIN-2361
Commit: 5629341122abe18d9576f71a0ea99ac83c327c21
Parents: a86569a
Author: xiefan46 <95...@qq.com>
Authored: Wed Jan 25 10:41:40 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jan 25 11:19:13 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/hdfs/HDFSResourceStore.java | 23 +++++++++++---------
1 file changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56293411/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 0ad6964..38acfb0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -18,7 +18,15 @@
package org.apache.kylin.storage.hdfs;
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,14 +40,7 @@ import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
public class HDFSResourceStore extends ResourceStore {
@@ -63,7 +64,7 @@ public class HDFSResourceStore extends ResourceStore {
int cut = metadataUrl.indexOf('@');
String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut);
String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
- if(!hdfsUrl.equals("hdfs"))
+ if (!hdfsUrl.equals("hdfs"))
throw new IOException("Can not create HDFSResourceStore. Url not match. Url:" + hdfsUrl);
metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
logger.info("meta dir name :" + metaDirName);
@@ -245,6 +246,8 @@ public class HDFSResourceStore extends ResourceStore {
}
private Path getRealHDFSPath(String resourcePath) {
+ if (resourcePath.equals("/"))
+ return this.hdfsMetaPath;
if (resourcePath.startsWith("/") && resourcePath.length() > 1)
resourcePath = resourcePath.substring(1, resourcePath.length());
return new Path(this.hdfsMetaPath, resourcePath);
[21/47] kylin git commit: KYLIN 2396 GUI for Percentile pre
aggregation implementation
Posted by bi...@apache.org.
KYLIN 2396 GUI for Percentile pre aggregation implementation
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b56bc622
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b56bc622
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b56bc622
Branch: refs/heads/KYLIN-2361
Commit: b56bc622086495f7d0047d01af57efc919b2ecce
Parents: 28673a8
Author: luguosheng <55...@qq.com>
Authored: Mon Jan 23 11:24:39 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 23 11:32:46 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 2 ++
webapp/app/partials/cubeDesigner/measures.html | 6 +++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 7c155fe..91c38f2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -355,6 +355,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.newMeasure.function.parameter.type= 'column';
$scope.newMeasure.function.returntype = "extendedcolumn(100)";
return;
+ }else if($scope.newMeasure.function.expression=='PERCENTILE'){
+ $scope.newMeasure.function.parameter.type= 'column';
}else{
$scope.nextParameterInit();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 4a48843..b7475b6 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -116,7 +116,7 @@
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Param Type</b></label>
<div class="col-xs-12 col-sm-6">
- <select class="form-control" ng-if="newMeasure.function.expression != 'COUNT'"
+ <select class="form-control" ng-if="newMeasure.function.expression != 'COUNT'&&newMeasure.function.expression != 'PERCENTILE'"
ng-init="newMeasure.function.parameter.type=(!!newMeasure.function.parameter.type)?newMeasure.function.parameter.type: 'column' "
chosen ng-model="newMeasure.function.parameter.type" required
ng-change="measureReturnTypeUpdate();"
@@ -124,8 +124,8 @@
<option value=""></option>
</select>
<span class="font-color-default"
- ng-if="newMeasure.function.expression == 'COUNT'"
- ng-init="newMeasure.function.parameter.type= 'constant' "><b> constant</b>
+ ng-if="newMeasure.function.expression == 'COUNT'||newMeasure.function.expression == 'PERCENTILE'"
+ ><b>{{newMeasure.function.parameter.type}}</b>
</span>
</div>
</div>
[02/47] kylin git commit: KYLIN-2410 let global dictionary use hadoop
conf in mapper/reducer context
Posted by bi...@apache.org.
KYLIN-2410 let global dictionary use hadoop conf in mapper/reducer context
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eddb695b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eddb695b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eddb695b
Branch: refs/heads/KYLIN-2361
Commit: eddb695ba13de282ada123f358aff723cb414316
Parents: d52bb8b
Author: Yang Li <li...@apache.org>
Authored: Thu Jan 19 08:56:22 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Jan 19 08:56:22 2017 +0800
----------------------------------------------------------------------
core-common/pom.xml | 5 +
.../apache/kylin/common/util/HadoopUtil.java | 146 ++++++++++++++++++
.../apache/kylin/dict/AppendTrieDictionary.java | 13 +-
.../kylin/dict/AppendTrieDictionaryChecker.java | 13 +-
.../org/apache/kylin/dict/CachedTreeMap.java | 7 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 6 +-
.../apache/kylin/dict/CachedTreeMapTest.java | 8 +-
.../apache/kylin/engine/mr/DFSFileTable.java | 1 +
.../kylin/engine/mr/DFSFileTableReader.java | 1 +
.../org/apache/kylin/engine/mr/HadoopUtil.java | 150 -------------------
.../org/apache/kylin/engine/mr/KylinMapper.java | 1 +
.../apache/kylin/engine/mr/KylinReducer.java | 1 +
.../kylin/engine/mr/SortedColumnDFSFile.java | 1 +
.../engine/mr/common/AbstractHadoopJob.java | 2 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../engine/mr/common/MapReduceExecutable.java | 2 +-
.../engine/mr/steps/CreateDictionaryJob.java | 2 +-
.../mr/steps/FactDistinctColumnsReducer.java | 2 +-
.../kylin/engine/mr/steps/InMemCuboidJob.java | 2 +-
.../engine/mr/steps/MergeStatisticsStep.java | 2 +-
.../steps/RowKeyDistributionCheckerMapper.java | 2 +-
.../engine/mr/steps/SaveStatisticsStep.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 2 +-
.../steps/FactDistinctColumnsReducerTest.java | 2 +-
.../HiveToBaseCuboidMapperPerformanceTest.java | 2 +-
.../engine/mr/steps/MergeCuboidJobTest.java | 2 +-
.../kylin/engine/mr/steps/NDCuboidJobTest.java | 2 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 2 +-
.../kylin/cube/ITDictionaryManagerTest.java | 2 +-
.../kylin/provision/BuildCubeWithEngine.java | 2 +-
.../apache/kylin/query/ITMassInQueryTest.java | 2 +-
.../kylin/storage/hbase/ITHdfsOpsTest.java | 2 +-
.../rest/controller/StreamingController.java | 2 +-
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../apache/kylin/rest/service/TableService.java | 2 +-
.../apache/kylin/source/hive/HiveMRInput.java | 2 +-
.../source/hive/HiveSourceTableLoader.java | 2 +-
.../apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../kylin/storage/hbase/HBaseConnection.java | 2 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 2 +-
.../storage/hbase/steps/DeprecatedGCStep.java | 2 +-
.../steps/HDFSPathGarbageCollectionStep.java | 2 +-
.../hbase/steps/SequenceFileCuboidWriter.java | 2 +-
.../storage/hbase/util/CubeMigrationCLI.java | 2 +-
.../storage/hbase/util/StorageCleanupJob.java | 2 +-
.../storage/hbase/steps/CreateHTableTest.java | 2 +-
.../hbase/steps/CubeHFileMapper2Test.java | 2 +-
.../steps/RangeKeyDistributionJobTest.java | 2 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 2 +-
.../apache/kylin/tool/MrJobInfoExtractor.java | 2 +-
.../apache/kylin/tool/StorageCleanupJob.java | 2 +-
.../org/apache/kylin/tool/util/ToolUtil.java | 2 +-
52 files changed, 216 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 25b10a7..016d470 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -78,6 +78,11 @@
<!-- Env & Test -->
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 0000000..390c209
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HadoopUtil {
+ @SuppressWarnings("unused")
+ private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
+ private static final transient ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
+
+ public static void setCurrentConfiguration(Configuration conf) {
+ hadoopConfig.set(conf);
+ }
+
+ public static Configuration getCurrentConfiguration() {
+ if (hadoopConfig.get() == null) {
+ Configuration conf = healSickConfig(new Configuration());
+ // do not cache this conf, or will affect following mr jobs
+ return conf;
+ }
+ Configuration conf = hadoopConfig.get();
+ return conf;
+ }
+
+ private static Configuration healSickConfig(Configuration conf) {
+ // https://issues.apache.org/jira/browse/KYLIN-953
+ if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
+ conf.set("hadoop.tmp.dir", "/tmp");
+ }
+ if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) {
+ conf.set("hbase.fs.tmp.dir", "/tmp");
+ }
+ return conf;
+ }
+
+ public static FileSystem getWorkingFileSystem() throws IOException {
+ return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ }
+
+ public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException {
+ Path workingPath = new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ return getFileSystem(workingPath, conf);
+ }
+
+ public static FileSystem getFileSystem(String path) throws IOException {
+ return getFileSystem(new Path(makeURI(path)));
+ }
+
+ public static FileSystem getFileSystem(Path path) throws IOException {
+ Configuration conf = getCurrentConfiguration();
+ return getFileSystem(path, conf);
+ }
+
+ public static FileSystem getFileSystem(Path path, Configuration conf) {
+ if (StringUtils.isBlank(path.toUri().getScheme()))
+ throw new IllegalArgumentException("Path must be qualified: " + path);
+
+ try {
+ return path.getFileSystem(conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static URI makeURI(String filePath) {
+ try {
+ return new URI(fixWindowsPath(filePath));
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
+ }
+ }
+
+ public static String fixWindowsPath(String path) {
+ // fix windows path
+ if (path.startsWith("file://") && !path.startsWith("file:///") && path.contains(":\\")) {
+ path = path.replace("file://", "file:///");
+ }
+ if (path.startsWith("file:///")) {
+ path = path.replace('\\', '/');
+ }
+ return path;
+ }
+
+ /**
+ * @param table the identifier of hive table, in format <db_name>.<table_name>
+ * @return a string array with 2 elements: {"db_name", "table_name"}
+ */
+ public static String[] parseHiveTableName(String table) {
+ int cut = table.indexOf('.');
+ String database = cut >= 0 ? table.substring(0, cut).trim() : "DEFAULT";
+ String tableName = cut >= 0 ? table.substring(cut + 1).trim() : table.trim();
+
+ return new String[] { database, tableName };
+ }
+
+ public static void deletePath(Configuration conf, Path path) throws IOException {
+ FileSystem fs = FileSystem.get(path.toUri(), conf);
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ }
+
+ public static byte[] toBytes(Writable writable) {
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bout);
+ writable.write(out);
+ out.close();
+ bout.close();
+ return bout.toByteArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index faffcc0..962686d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -51,6 +51,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.MetadataManager;
import org.slf4j.Logger;
@@ -1163,20 +1164,18 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
@Override
public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
//copy appendDict
- Configuration conf = new Configuration();
-
Path base = new Path(baseDir);
- FileSystem srcFs = FileSystem.get(base.toUri(), conf);
- Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base);
+ FileSystem srcFs = HadoopUtil.getFileSystem(base);
+ Path srcPath = CachedTreeMap.getLatestVersion(HadoopUtil.getCurrentConfiguration(), srcFs, base);
Path dstPath = new Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()));
logger.info("Copy appendDict from {} to {}", srcPath, dstPath);
- FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf);
+ FileSystem dstFs = HadoopUtil.getFileSystem(dstPath);
if (dstFs.exists(dstPath)) {
logger.info("Delete existing AppendDict {}", dstPath);
dstFs.delete(dstPath, true);
}
- FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf);
+ FileUtil.copy(srcFs, srcPath, dstFs, dstPath, false, true, HadoopUtil.getCurrentConfiguration());
// init new AppendTrieDictionary
AppendTrieDictionary newDict = new AppendTrieDictionary();
@@ -1194,7 +1193,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
@Override
public void readFields(DataInput in) throws IOException {
String baseDir = in.readUTF();
- Configuration conf = new Configuration();
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
try (FSDataInputStream input = CachedTreeMap.openLatestIndexInput(conf, baseDir)) {
int baseId = input.readInt();
int maxId = input.readInt();
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
index f231275..4b3817a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
@@ -17,16 +17,16 @@
*/
package org.apache.kylin.dict;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.kylin.common.util.HadoopUtil;
/**
* Created by sunyerui on 16/11/15.
@@ -34,9 +34,8 @@ import java.util.List;
public class AppendTrieDictionaryChecker {
public boolean runChecker(String baseDir) throws IOException {
- Configuration conf = new Configuration();
Path basePath = new Path(baseDir);
- FileSystem fs = FileSystem.get(basePath.toUri(), conf);
+ FileSystem fs = HadoopUtil.getFileSystem(basePath);
List<Path> sliceList = new ArrayList<>();
List<Path> corruptedSliceList = new ArrayList<>();
listDictSlicePath(fs, fs.getFileStatus(basePath), sliceList);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index cc23261..ee69df7 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -145,12 +146,12 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
this.keepAppend = true;
this.maxVersions = maxVersions;
this.versionTTL = versionTTL;
- this.conf = new Configuration();
+ this.conf = HadoopUtil.getCurrentConfiguration();
if (basePath.endsWith("/")) {
basePath = basePath.substring(0, basePath.length()-1);
}
this.baseDir = new Path(basePath);
- this.fs = FileSystem.get(baseDir.toUri(), conf);
+ this.fs = HadoopUtil.getFileSystem(baseDir, conf);
if (!fs.exists(baseDir)) {
fs.mkdirs(baseDir);
}
@@ -447,7 +448,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
public static FSDataInputStream openLatestIndexInput(Configuration conf, String baseDir) throws IOException {
Path basePath = new Path(baseDir);
- FileSystem fs = FileSystem.get(basePath.toUri(), conf);
+ FileSystem fs = HadoopUtil.getFileSystem(basePath, conf);
Path indexPath = new Path(getLatestVersion(conf, fs, basePath), ".index");
return fs.open(indexPath, 8 * 1024 * 1024);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 18913d0..e2af338 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -40,9 +40,8 @@ import java.util.TreeMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Before;
@@ -71,10 +70,9 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
}
public static void cleanup() {
- Configuration conf = new Configuration();
Path basePath = new Path(BASE_DIR);
try {
- FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+ HadoopUtil.getFileSystem(basePath).delete(basePath, true);
} catch (IOException e) {}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index 64eb13c..ccf6e24 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -31,11 +31,11 @@ import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.common.util.HadoopUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Test;
@@ -118,10 +118,9 @@ public class CachedTreeMapTest {
public static final String workingDir = "/tmp/kylin_cachedtreemap_test/working";
private static void cleanup() {
- Configuration conf = new Configuration();
Path basePath = new Path(baseDir);
try {
- FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+ HadoopUtil.getFileSystem(basePath).delete(basePath, true);
} catch (IOException e) {}
VALUE_WRITE_ERROR_TOGGLE = false;
}
@@ -292,8 +291,7 @@ public class CachedTreeMapTest {
// move version dir to base dir, to simulate the older format
Path versionPath = new Path(map.getLatestVersion());
Path tmpVersionPath = new Path(versionPath.getParent().getParent(), versionPath.getName());
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(versionPath.toUri(), conf);
+ FileSystem fs = HadoopUtil.getFileSystem(versionPath);
fs.rename(versionPath, tmpVersionPath);
fs.delete(new Path(baseDir), true);
fs.rename(tmpVersionPath, new Path(baseDir));
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index ee932ac..19bc021 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.source.ReadableTable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index 847f4bf..8c6b5f5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.source.ReadableTable.TableReader;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
deleted file mode 100644
index 7665350..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.io.Writable;
-import org.apache.kylin.common.KylinConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HadoopUtil {
- @SuppressWarnings("unused")
- private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
- private static final transient ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
-
- public static void setCurrentConfiguration(Configuration conf) {
- hadoopConfig.set(conf);
- }
-
- public static Configuration getCurrentConfiguration() {
- if (hadoopConfig.get() == null) {
- Configuration conf = healSickConfig(new Configuration());
- // do not cache this conf, or will affect following mr jobs
- return conf;
- }
- Configuration conf = hadoopConfig.get();
- return conf;
- }
-
- private static Configuration healSickConfig(Configuration conf) {
- // why we have this hard code?
- conf.set(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, "8");
-
- // https://issues.apache.org/jira/browse/KYLIN-953
- if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
- conf.set("hadoop.tmp.dir", "/tmp");
- }
- if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) {
- conf.set("hbase.fs.tmp.dir", "/tmp");
- }
- return conf;
- }
-
- public static FileSystem getWorkingFileSystem() throws IOException {
- return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
- }
-
- public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException {
- Path workingPath = new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
- return getFileSystem(workingPath, conf);
- }
-
- public static FileSystem getFileSystem(String path) throws IOException {
- return getFileSystem(new Path(makeURI(path)));
- }
-
- public static FileSystem getFileSystem(Path path) throws IOException {
- Configuration conf = getCurrentConfiguration();
- return getFileSystem(path, conf);
- }
-
- public static FileSystem getFileSystem(Path path, Configuration conf) {
- if (StringUtils.isBlank(path.toUri().getScheme()))
- throw new IllegalArgumentException("Path must be qualified: " + path);
-
- try {
- return path.getFileSystem(conf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static URI makeURI(String filePath) {
- try {
- return new URI(fixWindowsPath(filePath));
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
- }
- }
-
- public static String fixWindowsPath(String path) {
- // fix windows path
- if (path.startsWith("file://") && !path.startsWith("file:///") && path.contains(":\\")) {
- path = path.replace("file://", "file:///");
- }
- if (path.startsWith("file:///")) {
- path = path.replace('\\', '/');
- }
- return path;
- }
-
- /**
- * @param table the identifier of hive table, in format <db_name>.<table_name>
- * @return a string array with 2 elements: {"db_name", "table_name"}
- */
- public static String[] parseHiveTableName(String table) {
- int cut = table.indexOf('.');
- String database = cut >= 0 ? table.substring(0, cut).trim() : "DEFAULT";
- String tableName = cut >= 0 ? table.substring(cut + 1).trim() : table.trim();
-
- return new String[] { database, tableName };
- }
-
- public static void deletePath(Configuration conf, Path path) throws IOException {
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- }
-
- public static byte[] toBytes(Writable writable) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bout);
- writable.write(out);
- out.close();
- bout.close();
- return bout.toByteArray();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 2b564e9..1595bdd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index cb2d7a7..d428757 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index 62c309a..d3f5cdc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -24,6 +24,7 @@ import java.util.Comparator;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.dict.ByteComparator;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.metadata.datatype.DataType;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 1540acd..567c1d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -59,12 +59,12 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 4011915..dbe4554 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -43,6 +43,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.SumHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -52,7 +53,6 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index be11e39..f887c4c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.constant.JobStepStatusEnum;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index b27d722..95d8cb1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.cli.DictionaryGeneratorCLI;
import org.apache.kylin.dict.DictionaryProvider;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.SortedColumnDFSFile;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 3c7f951..711d991 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -38,12 +38,12 @@ import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IDictionaryBuilder;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index 1612866..7706bac 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -30,12 +30,12 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.ByteArrayWritable;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 54666d0..04d8231 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -37,10 +37,10 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.job.exception.ExecuteException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index ee8da6b..a2efe04 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.KylinMapper;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 6120270..2671042 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -27,9 +27,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dcc9190..dc80399 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -24,11 +24,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index f6f790e..c971cef 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -26,7 +26,7 @@ import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
index c1f50a8..9e03493 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
index 97b1ef2..2e2ebf9 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
@@ -26,8 +26,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
index 10ebd2b..989ed72 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
@@ -25,8 +25,8 @@ import java.io.File;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 3a664fc..d6790aa 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -45,7 +46,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.BatchCubingJobBuilder2;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BaseCuboidBuilder;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
index 188a97a..458703a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
@@ -27,6 +27,7 @@ import java.io.PrintWriter;
import java.util.Set;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
@@ -34,7 +35,6 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
import org.apache.kylin.engine.mr.DFSFileTable;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.junit.After;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 62978db..e02bf19 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -47,7 +48,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.DeployUtil;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 28cdb67..8859329 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -29,7 +29,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.dbunit.dataset.ITable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 605b79e..01a71bf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 0ced9ad..407ee2e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -23,8 +23,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.rest.exception.BadRequestException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 7dd7b0c..614f0c9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index e3df60a..d4cb854 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -30,7 +30,7 @@ import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index bd53f9a..d665dc7 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -31,12 +31,12 @@ import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BufferedLogger;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 286ffac..87edfe4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,8 +26,8 @@ import java.util.Set;
import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 2f4fded..c7987f2 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -31,10 +31,10 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.common.BatchConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..335bfe7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.StorageException;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 2a12984..6217350 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -48,7 +48,7 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index fbe64d9..eacff9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -32,8 +32,8 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
index 89baf95..4c0747e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.ExecuteException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 1d66d3e..5e6ad34 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -25,10 +25,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.ByteArrayWritable;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.steps.KVGTRecordWriter;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 31864f6..68c0a39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -47,6 +47,7 @@ import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -55,7 +56,6 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index d1a74ad..62af2c9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -44,13 +44,13 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
index f994886..71079c7 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
@@ -23,8 +23,8 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index 77b18e2..f36a62c 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.MeasureCodec;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
index 70e1ac7..1d75cfb 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
@@ -25,8 +25,8 @@ import java.io.File;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 5269195..c8bff89 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -47,6 +47,7 @@ import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -55,7 +56,6 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index 1050bbe..ea19885 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -37,8 +37,8 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index b3e2ec0..b48c076 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -45,11 +45,11 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
index 1312ca4..c41d6a8 100644
--- a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
+++ b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
import com.google.common.collect.Maps;
[11/47] kylin git commit: minor,
fix false positive warning in test output
Posted by bi...@apache.org.
minor, fix false positive warning in test output
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39
Branch: refs/heads/KYLIN-2361
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 20 08:13:11 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/AbstractKylinTestCase.java | 7 ++++++-
.../java/org/apache/kylin/dict/lookup/SnapshotManager.java | 6 ++++++
.../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++++++
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
examples/test_case_data/localmeta/cube_desc/ssb.json | 4 ++--
examples/test_case_data/localmeta/cube_desc/ssb_cube1.json | 4 ++--
examples/test_case_data/localmeta/cube_desc/ssb_cube2.json | 4 ++--
examples/test_case_data/localmeta/cube_desc/ssb_cube3.json | 4 ++--
.../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
.../localmeta/cube_desc/test_streaming_table_cube_desc.json | 2 +-
examples/test_case_data/localmeta/kylin.properties | 2 +-
examples/test_case_data/sandbox/kylin.properties | 2 +-
kylin-it/pom.xml | 2 +-
13 files changed, 33 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
public static final String[] SERVICES_WITH_CACHE = { //
"org.apache.kylin.cube.CubeManager", //
"org.apache.kylin.cube.CubeDescManager", //
+ "org.apache.kylin.dict.lookup.SnapshotManager", //
+ "org.apache.kylin.dict.DictionaryManager", //
"org.apache.kylin.storage.hybrid.HybridManager", //
"org.apache.kylin.metadata.realization.RealizationRegistry", //
"org.apache.kylin.metadata.project.ProjectManager", //
"org.apache.kylin.metadata.MetadataManager", //
+ "org.apache.kylin.metadata.cachesync.Broadcaster", //
+ "org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
"org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
- "org.apache.kylin.job.manager.ExecutableManager", //
+ "org.apache.kylin.job.execution.ExecutableManager", //
+ "org.apache.kylin.job.dao.ExecutableDao" //
};
public abstract void createTestMetadata() throws Exception;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
return r;
}
+ public static void clearCache() {
+ synchronized (SERVICE_CACHE) {
+ SERVICE_CACHE.clear();
+ }
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
return r;
}
+ public static void clearCache() {
+ synchronized (CACHE) {
+ CACHE.clear();
+ }
+ }
+
private ExecutableDao(KylinConfig config) {
logger.info("Using metadata url: " + config);
this.store = MetadataManager.getInstance(config).getStore();
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index d00c490..17b644d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -83,7 +83,7 @@ public class Broadcaster {
}
// call Broadcaster.getInstance().notifyClearAll() to clear cache
- static void clearCache() {
+ public static void clearCache() {
synchronized (CACHE) {
CACHE.clear();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index a13ac53..280fc82 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -183,7 +183,7 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ "kylin.storage.hbase.compression-codec" : "lz4",
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
index 5986057..861e67d 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
@@ -149,7 +149,7 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ "kylin.storage.hbase.compression-codec" : "lz4",
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
index fc75ea8..c2bf3c0 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
@@ -149,7 +149,7 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ "kylin.storage.hbase.compression-codec" : "lz4",
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
index 4509116..d45b6a7 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
@@ -137,7 +137,7 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ "kylin.storage.hbase.compression-codec" : "lz4",
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 9aeaa2e..25b66f2 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -287,8 +287,8 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties": {
- "kylin.job.cubing.inmem.sampling.hll.precision": "16",
- "kylin.hive.create.flat.table.method": "2"
+ "kylin.job.sampling-hll-precision": "16",
+ "kylin.source.hive.create-flat-table-method": "2"
},
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index 640b91c..0065b02 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -106,7 +106,7 @@
} ],
"override_kylin_properties": {
"kylin.cube.algorithm": "inmem",
- "kylin.cube.building.segment.max": "5"
+ "kylin.cube.max-building-segments": "5"
},
"notify_list" : [ ],
"status_need_notify" : [ ],
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index f4c6772..e89c767 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -133,4 +133,4 @@ kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
-kylin.cube.engine.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
+kylin.engine.provider.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 06f8e4b..6c512dc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -25,7 +25,7 @@ kylin.server.mode=all
kylin.storage.hbase.owner-tag=whoami@kylin.apache.org
# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.server.cluster-servers=localhost:7070
+#kylin.server.cluster-servers=localhost:7070
# Display timezone on UI,format like[GMT+N or GMT-N]
kylin.web.timezone=GMT+8
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..f972b4b 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -282,7 +282,7 @@
<value>file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</value>
</property>
</systemProperties>
- <argLine>-Xms1G -Xmx2G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
+ <argLine>-Xms1G -Xmx2G -XX:PermSize=128M -XX:MaxPermSize=512M -Dkylin.server.cluster-servers=localhost:7070</argLine>
</configuration>
</plugin>
<plugin>
[46/47] kylin git commit: KYLIN-2361 Upgrade Tomcat 8.5.9
Posted by bi...@apache.org.
KYLIN-2361 Upgrade Tomcat 8.5.9
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b60ac6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b60ac6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b60ac6a
Branch: refs/heads/KYLIN-2361
Commit: 2b60ac6a42741ca70d63e6680a0fbe9aeed7d46e
Parents: a058bfb
Author: Billy Liu <bi...@apache.org>
Authored: Wed Jan 18 17:19:04 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sat Feb 4 11:42:02 2017 +0800
----------------------------------------------------------------------
build/script/download-tomcat.sh | 8 ++++----
pom.xml | 2 +-
.../java/org/apache/kylin/rest/DebugTomcat.java | 16 +++++++++-------
.../kylin/ext/CustomizedWebappClassloader.java | 5 ++---
4 files changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index b3aa509..bdfe351 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,19 +27,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-tomcat_pkg_version="7.0.69"
-tomcat_pkg_md5="10a071e5169a1a8b14ff35a0ad181052"
+tomcat_pkg_version="8.5.9"
+tomcat_pkg_md5="b41270a64b7774c964e4bec813eea2ed"
if [ ! -f "build/apache-tomcat-${tomcat_pkg_version}.tar.gz" ]
then
echo "no binary file found"
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "Download tomcat failed"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "Download tomcat failed"
else
if [ `md5cmd build/apache-tomcat-${tomcat_pkg_version}.tar.gz | awk '{print $1}'` != "${tomcat_pkg_md5}" ]
then
echo "md5 check failed"
rm build/apache-tomcat-${tomcat_pkg_version}.tar.gz
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "download tomcat failed"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "download tomcat failed"
fi
fi
unalias md5cmd
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bf33e07..b82eee2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
<cglib.version>3.2.4</cglib.version>
<supercsv.version>2.4.0</supercsv.version>
<cors.version>2.5</cors.version>
- <tomcat.version>7.0.69</tomcat.version>
+ <tomcat.version>8.5.9</tomcat.version>
<t-digest.version>3.1</t-digest.version>
<!-- REST Service -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 3461e1d..1b47f79 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -21,11 +21,13 @@ package org.apache.kylin.rest;
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
-import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.JarScanFilter;
+import org.apache.tomcat.JarScanType;
+import org.apache.tomcat.util.descriptor.web.ErrorPage;
import java.io.File;
import java.lang.reflect.Field;
@@ -127,12 +129,12 @@ public class DebugTomcat {
notFound.setLocation("/index.html");
webContext.addErrorPage(notFound);
webContext.addWelcomeFile("index.html");
-// webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
-// @Override
-// public boolean check(JarScanType arg0, String arg1) {
-// return false;
-// }
-// });
+ webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
+ @Override
+ public boolean check(JarScanType arg0, String arg1) {
+ return false;
+ }
+ });
// tomcat start
tomcat.start();
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index 816601f..f241865 100644
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -45,7 +45,7 @@ public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
* @param name class name
* @return true if the class should be filtered
*/
- protected boolean filter(String name) {
+ protected boolean filter(String name, boolean isClassName) {
if (name == null)
return false;
@@ -62,7 +62,6 @@ public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
return true;
}
- //return super.filter(name, isClassName);
- return false;
+ return super.filter(name, isClassName);
}
}
[30/47] kylin git commit: Add logging info
Posted by bi...@apache.org.
Add logging info
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/971bf800
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/971bf800
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/971bf800
Branch: refs/heads/KYLIN-2361
Commit: 971bf80009c446a873ddc0927151eb1e68d36e27
Parents: e217ae8
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 23 11:49:27 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/kafka/job/MergeOffsetStep.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/971bf800/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
index 9cadd72..914fca2 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import com.google.common.base.Preconditions;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
public class MergeOffsetStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(MergeOffsetStep.class);
+
public MergeOffsetStep() {
super();
}
@@ -46,12 +48,15 @@ public class MergeOffsetStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+ final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams());
+ final CubeSegment segment = cube.getSegmentById(segmentId);
+ Preconditions.checkNotNull(segment, "Cube segment '" + segmentId + "' not found.");
List<CubeSegment> mergingSegs = cube.getMergingSegments(segment);
- Collections.sort(mergingSegs);
+ Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist.");
+ Collections.sort(mergingSegs);
final CubeSegment first = mergingSegs.get(0);
final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);
[20/47] kylin git commit: minor,
add config kylin.cube.allow-appear-in-multiple-projects
Posted by bi...@apache.org.
minor, add config kylin.cube.allow-appear-in-multiple-projects
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6aaf6676
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6aaf6676
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6aaf6676
Branch: refs/heads/KYLIN-2361
Commit: 6aaf6676b5e904260f16d1222bfe99013a095963
Parents: 2c4591a
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Jan 23 11:31:38 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jan 23 11:31:38 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 9 ++++++---
.../src/main/java/org/apache/kylin/engine/mr/CubingJob.java | 7 ++++++-
2 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 74903d5..05df177 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -181,7 +181,7 @@ abstract public class KylinConfigBase implements Serializable {
if (!root.endsWith("/")) {
root += "/";
}
-
+
// make sure path qualified
if (!root.contains("://")) {
if (!root.startsWith("/"))
@@ -189,7 +189,7 @@ abstract public class KylinConfigBase implements Serializable {
else
root = "hdfs://" + root;
}
-
+
return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
@@ -320,6 +320,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
+ public boolean allowCubeAppearInMultipleProjects() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.allow-appear-in-multiple-projects", "false"));
+ }
+
// ============================================================================
// JOB
// ============================================================================
@@ -767,7 +771,6 @@ abstract public class KylinConfigBase implements Serializable {
return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0"));
}
-
public int getSparkMinPartition() {
return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", "1"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 64e9edb..5aa7d72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -84,7 +84,12 @@ public class CubingJob extends DefaultChainedExecutable {
if (projList == null || projList.size() == 0) {
throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
} else if (projList.size() >= 2) {
- throw new RuntimeException("Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! ");
+ String msg = "Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! ";
+ if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
+ throw new RuntimeException(msg);
+ } else {
+ logger.warn(msg);
+ }
}
CubingJob result = new CubingJob();
[27/47] kylin git commit: KYLIN-2413 update test conf files
Posted by bi...@apache.org.
KYLIN-2413 update test conf files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d10394
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d10394
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d10394
Branch: refs/heads/KYLIN-2361
Commit: 87d103948fdf8951941c9a7fd11213dd569f8aa5
Parents: eb73da7
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jan 23 13:12:36 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 23 13:18:24 2017 +0800
----------------------------------------------------------------------
dev-support/sync_hbase_cdh_branches.sh | 15 +--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 ----
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 ++++
.../sandbox/capacity-scheduler.xml | 17 +--
examples/test_case_data/sandbox/core-site.xml | 35 ++++-
.../test_case_data/sandbox/hadoop-policy.xml | 2 +-
.../test_case_data/sandbox/hbase-policy.xml | 2 +-
examples/test_case_data/sandbox/hbase-site.xml | 100 +++++++++++++--
examples/test_case_data/sandbox/hdfs-site.xml | 84 +++++++-----
examples/test_case_data/sandbox/hive-site.xml | 89 ++++++++-----
examples/test_case_data/sandbox/mapred-site.xml | 57 ++++++---
examples/test_case_data/sandbox/yarn-site.xml | 127 ++++++++++++++++---
12 files changed, 404 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/sync_hbase_cdh_branches.sh
----------------------------------------------------------------------
diff --git a/dev-support/sync_hbase_cdh_branches.sh b/dev-support/sync_hbase_cdh_branches.sh
index 472f402..592053a 100755
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -37,24 +37,17 @@ trap 'error ${LINENO} ${?}' ERR
# ============================================================================
git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh5.7
+git checkout apache/$base-hbase0.98
git format-patch -1
git checkout apache/$base
git checkout -b tmp
git reset apache/$base --hard
-git am -3 --ignore-whitespace 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-mvn clean compile -DskipTests
-git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git am -3 --ignore-whitespace 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
mvn clean compile -DskipTests
-git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git push apache tmp:$base-hbase0.98 -f
+rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
# clean up
git checkout master
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
deleted file mode 100755
index f7780dd..0000000
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_4_0_0_169.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
new file mode 100755
index 0000000..2a3d24b
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index 7cb985c..e042aa5 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -47,16 +47,6 @@
</property>
<property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
- <value>-1</value>
- </property>
-
- <property>
<name>yarn.scheduler.capacity.root.acl_administer_queue</name>
<value>*</value>
</property>
@@ -67,11 +57,6 @@
</property>
<property>
- <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
- <value></value>
- </property>
-
- <property>
<name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
<value>*</value>
</property>
@@ -111,4 +96,4 @@
<value>default</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..7660a7e 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -19,7 +19,6 @@
<property>
<name>fs.defaultFS</name>
<value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
</property>
<property>
@@ -39,7 +38,7 @@
<property>
<name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <value>*</value>
</property>
<property>
@@ -49,7 +48,7 @@
<property>
<name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <value>*</value>
</property>
<property>
@@ -68,13 +67,23 @@
</property>
<property>
+ <name>hadoop.proxyuser.hdfs.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hdfs.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
<name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
- <value>*</value>
+ <value>sandbox.hortonworks.com</value>
</property>
<property>
@@ -123,8 +132,15 @@
</property>
<property>
+ <name>hadoop.security.key.provider.path</name>
+ <value></value>
+ </property>
+
+ <property>
<name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ <value>
+ org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec
+ </value>
</property>
<property>
@@ -162,4 +178,9 @@
<value>false</value>
</property>
-</configuration>
+ <property>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf/topology_script.py</value>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml
index a590979..20b894e 100644
--- a/examples/test_case_data/sandbox/hadoop-policy.xml
+++ b/examples/test_case_data/sandbox/hadoop-policy.xml
@@ -71,4 +71,4 @@
<value>hadoop</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml
index 178b559..e0f2da6 100644
--- a/examples/test_case_data/sandbox/hbase-policy.xml
+++ b/examples/test_case_data/sandbox/hbase-policy.xml
@@ -31,4 +31,4 @@
<value>*</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 734908e..568de2e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -22,8 +22,33 @@
</property>
<property>
+ <name>hbase.bucketcache.ioengine</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>hbase.bucketcache.percentage.in.combinedcache</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>hbase.bucketcache.size</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>hbase.bulkload.staging.dir</name>
+ <value>/apps/hbase/staging</value>
+ </property>
+
+ <property>
<name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
+ <value>1048576</value>
+ </property>
+
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>35</value>
</property>
<property>
@@ -38,12 +63,19 @@
<property>
<name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <value>org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <value>
+ org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor
+ </value>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.regionserver.classes</name>
+ <value></value>
</property>
<property>
@@ -87,6 +119,11 @@
</property>
<property>
+ <name>hbase.hstore.compaction.max</name>
+ <value>10</value>
+ </property>
+
+ <property>
<name>hbase.hstore.compactionThreshold</name>
<value>3</value>
</property>
@@ -103,32 +140,42 @@
<property>
<name>hbase.master.info.port</name>
- <value>60010</value>
+ <value>16010</value>
</property>
<property>
<name>hbase.master.port</name>
- <value>60000</value>
+ <value>16000</value>
</property>
<property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
+ <name>hbase.region.server.rpc.scheduler.factory.class</name>
+ <value></value>
</property>
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <name>hbase.regionserver.global.memstore.size</name>
<value>0.4</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <value>30</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <value>16030</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.port</name>
+ <value>16020</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.wal.codec</name>
+ <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value>
</property>
<property>
@@ -137,11 +184,26 @@
</property>
<property>
+ <name>hbase.rpc.controllerfactory.class</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>hbase.rpc.engine</name>
+ <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
+ </property>
+
+ <property>
<name>hbase.rpc.protection</name>
<value>PRIVACY</value>
</property>
<property>
+ <name>hbase.rpc.timeout</name>
+ <value>90000</value>
+ </property>
+
+ <property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
@@ -158,7 +220,7 @@
<property>
<name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <value>/tmp/hbase-${user.name}</value>
</property>
<property>
@@ -178,12 +240,22 @@
<property>
<name>hfile.block.cache.size</name>
- <value>0.40</value>
+ <value>0.4</value>
+ </property>
+
+ <property>
+ <name>phoenix.functions.allowUserDefinedFunctions</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>phoenix.query.timeoutMs</name>
+ <value>60000</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <value>60000</value>
</property>
<property>
@@ -191,4 +263,4 @@
<value>/hbase-unsecure</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1175fff..1d9040a 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -18,12 +18,7 @@
<property>
<name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
+ <value>true</value>
</property>
<property>
@@ -47,11 +42,21 @@
</property>
<property>
+ <name>dfs.client.retry.policy.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>dfs.cluster.administrators</name>
<value>hdfs</value>
</property>
<property>
+ <name>dfs.content-summary.limit</name>
+ <value>5000</value>
+ </property>
+
+ <property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:50010</value>
</property>
@@ -64,7 +69,6 @@
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/hdfs/data</value>
- <final>true</final>
</property>
<property>
@@ -80,7 +84,6 @@
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
- <final>true</final>
</property>
<property>
@@ -104,13 +107,18 @@
</property>
<property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <name>dfs.encrypt.data.transfer.cipher.suites</name>
+ <value>AES/CTR/NoPadding</value>
+ </property>
+
+ <property>
+ <name>dfs.encryption.key.provider.uri</name>
+ <value></value>
</property>
<property>
@@ -150,7 +158,12 @@
<property>
<name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.audit.log.async</name>
+ <value>true</value>
</property>
<property>
@@ -184,6 +197,11 @@
</property>
<property>
+ <name>dfs.namenode.fslock.fair</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
@@ -191,7 +209,6 @@
<property>
<name>dfs.namenode.http-address</name>
<value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
</property>
<property>
@@ -200,9 +217,13 @@
</property>
<property>
+ <name>dfs.namenode.inode.attributes.provider.class</name>
+ <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
+ </property>
+
+ <property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/hdfs/namenode</value>
- <final>true</final>
</property>
<property>
@@ -211,8 +232,13 @@
</property>
<property>
+ <name>dfs.namenode.rpc-address</name>
+ <value>sandbox.hortonworks.com:8020</value>
+ </property>
+
+ <property>
<name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>0.999</value>
</property>
<property>
@@ -236,16 +262,6 @@
</property>
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
- </property>
-
- <property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
- </property>
-
- <property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
@@ -257,7 +273,7 @@
<property>
<name>dfs.replication</name>
- <value>1</value>
+ <value>3</value>
</property>
<property>
@@ -268,13 +284,11 @@
<property>
<name>dfs.support.append</name>
<value>true</value>
- <final>true</final>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
- <final>true</final>
</property>
<property>
@@ -282,4 +296,14 @@
<value>022</value>
</property>
-</configuration>
+ <property>
+ <name>nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>nfs.file.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index 1e78107..a8c210e 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -22,11 +22,46 @@
</property>
<property>
+ <name>atlas.cluster.name</name>
+ <value>Sandbox</value>
+ </property>
+
+ <property>
+ <name>atlas.hook.hive.maxThreads</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>atlas.hook.hive.minThreads</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>atlas.hook.hive.synchronous</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>atlas.rest.address</name>
+ <value>http://sandbox.hortonworks.com:21000</value>
+ </property>
+
+ <property>
+ <name>datanucleus.autoCreateSchema</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>datanucleus.cache.level2.type</name>
<value>none</value>
</property>
<property>
+ <name>datanucleus.fixedDatastore</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>hive.auto.convert.join</name>
<value>true</value>
</property>
@@ -38,7 +73,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>1000000000</value>
+ <value>357913941</value>
</property>
<property>
@@ -127,6 +162,16 @@
</property>
<property>
+ <name>hive.default.fileformat</name>
+ <value>TextFile</value>
+ </property>
+
+ <property>
+ <name>hive.default.fileformat.managed</name>
+ <value>TextFile</value>
+ </property>
+
+ <property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>
@@ -162,11 +207,6 @@
</property>
<property>
- <name>hive.exec.failure.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
<name>hive.exec.max.created.files</name>
<value>100000</value>
</property>
@@ -197,6 +237,11 @@
</property>
<property>
+ <name>hive.exec.orc.encoding.strategy</name>
+ <value>SPEED</value>
+ </property>
+
+ <property>
<name>hive.exec.parallel</name>
<value>false</value>
</property>
@@ -207,16 +252,6 @@
</property>
<property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>67108864</value>
</property>
@@ -262,11 +297,6 @@
</property>
<property>
- <name>hive.heapsize</name>
- <value>250</value>
- </property>
-
- <property>
<name>hive.limit.optimize.enable</name>
<value>true</value>
</property>
@@ -478,7 +508,7 @@
<property>
<name>hive.prewarm.numcontainers</name>
- <value>10</value>
+ <value>3</value>
</property>
<property>
@@ -488,7 +518,7 @@
<property>
<name>hive.security.authorization.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -508,7 +538,7 @@
<property>
<name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value>
</property>
<property>
@@ -533,12 +563,7 @@
<property>
<name>hive.server2.enable.doAs</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.server2.enable.impersonation</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -548,7 +573,7 @@
<property>
<name>hive.server2.logging.operation.log.location</name>
- <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
+ <value>/tmp/hive/operation_logs</value>
</property>
<property>
@@ -653,7 +678,7 @@
<property>
<name>hive.tez.container.size</name>
- <value>250</value>
+ <value>1024</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index e90f594..be470f9 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -27,13 +27,13 @@
</property>
<property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <name>mapred.job.map.memory.mb</name>
+ <value>250</value>
</property>
<property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <name>mapred.job.reduce.memory.mb</name>
+ <value>250</value>
</property>
<property>
@@ -48,7 +48,9 @@
<property>
<name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ <value>
+ LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64
+ </value>
</property>
<property>
@@ -58,7 +60,9 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>
+ $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure
+ </value>
</property>
<property>
@@ -77,14 +81,18 @@
</property>
<property>
+ <name>mapreduce.job.counters.max</name>
+ <value>130</value>
+ </property>
+
+ <property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
</property>
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
+ <value>0.05</value>
</property>
<property>
@@ -108,13 +116,28 @@
</property>
<property>
+ <name>mapreduce.jobhistory.recovery.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.class</name>
+ <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
+ <value>/hadoop/mapreduce/jhs</value>
+ </property>
+
+ <property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>sandbox.hortonworks.com:19888</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx1228m</value>
</property>
<property>
@@ -124,7 +147,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>1536</value>
</property>
<property>
@@ -159,7 +182,7 @@
<property>
<name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
+ <value>-Xmx1638m</value>
</property>
<property>
@@ -169,7 +192,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <value>2048</value>
</property>
<property>
@@ -219,7 +242,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
+ <value>859</value>
</property>
<property>
@@ -234,7 +257,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx819m -Dhdp.version=${hdp.version}</value>
</property>
<property>
@@ -244,7 +267,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
+ <value>1024</value>
</property>
<property>
@@ -252,4 +275,4 @@
<value>/user</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 8256158..ebdf44a 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -18,7 +18,7 @@
<property>
<name>hadoop.registry.rm.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -28,22 +28,29 @@
<property>
<name>yarn.acl.enable</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
<name>yarn.admin.acl</name>
- <value></value>
+ <value>*</value>
</property>
<property>
<name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
+ <value>
+ $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*
+ </value>
+ </property>
+
+ <property>
+ <name>yarn.authorization-provider</name>
+ <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
</property>
<property>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
+ <value>120000</value>
</property>
<property>
@@ -72,6 +79,11 @@
</property>
<property>
+ <name>yarn.node-labels.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>yarn.node-labels.fs-store.retry-policy-spec</name>
<value>2000, 500</value>
</property>
@@ -82,11 +94,6 @@
</property>
<property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
<name>yarn.nodemanager.address</name>
<value>0.0.0.0:45454</value>
</property>
@@ -98,7 +105,7 @@
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
+ <value>mapreduce_shuffle,spark_shuffle</value>
</property>
<property>
@@ -107,6 +114,11 @@
</property>
<property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
+ <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.bind-host</name>
<value>0.0.0.0</value>
</property>
@@ -148,7 +160,7 @@
<property>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
+ <value>120000</value>
</property>
<property>
@@ -243,12 +255,12 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
+ <value>7168</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
+ <value>80</value>
</property>
<property>
@@ -337,6 +349,11 @@
</property>
<property>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>yarn.resourcemanager.state-store.max-completed-applications</name>
<value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
@@ -368,7 +385,7 @@
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
+ <value>sandbox.hortonworks.com:8090</value>
</property>
<property>
@@ -408,7 +425,7 @@
<property>
<name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <value>sandbox.hortonworks.com:2181</value>
</property>
<property>
@@ -433,12 +450,22 @@
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
+ <value>7168</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>3</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
</property>
<property>
@@ -467,6 +494,41 @@
</property>
<property>
+ <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
+ <value>/ats/active/</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
+ <value>3600</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
+ <value>/ats/done/</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
+ <value>org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
+ <value>604800</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
+ <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
+ </property>
+
+ <property>
<name>yarn.timeline-service.generic-application-history.store-class</name>
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
@@ -482,6 +544,11 @@
</property>
<property>
+ <name>yarn.timeline-service.leveldb-state-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ </property>
+
+ <property>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
<value>/hadoop/yarn/timeline</value>
</property>
@@ -507,8 +574,23 @@
</property>
<property>
+ <name>yarn.timeline-service.plugin.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.recovery.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.state-store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
+ </property>
+
+ <property>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
</property>
<property>
@@ -522,6 +604,11 @@
</property>
<property>
+ <name>yarn.timeline-service.version</name>
+ <value>1.5</value>
+ </property>
+
+ <property>
<name>yarn.timeline-service.webapp.address</name>
<value>sandbox.hortonworks.com:8188</value>
</property>
@@ -531,4 +618,4 @@
<value>sandbox.hortonworks.com:8190</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
[23/47] kylin git commit: add back calcite linq4j
Posted by bi...@apache.org.
add back calcite linq4j
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68e87f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68e87f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68e87f08
Branch: refs/heads/KYLIN-2361
Commit: 68e87f08255caf33dd7946f43b3393372a73c68b
Parents: b56bc62
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Jan 23 11:54:54 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jan 23 11:54:54 2017 +0800
----------------------------------------------------------------------
pom.xml | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/68e87f08/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 57b7752..e590fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-linq4j</artifactId>
+ <version>${calcite.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
<version>${avatica.version}</version>
[39/47] kylin git commit: KYLIN-2374 minor,
rename zookeeper config properties
Posted by bi...@apache.org.
KYLIN-2374 minor,rename zookeeper config properties
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2a59c43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2a59c43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2a59c43
Branch: refs/heads/KYLIN-2361
Commit: a2a59c4303a29de308beb7b5c8300f78023313dd
Parents: 5629341
Author: xiefan46 <95...@qq.com>
Authored: Tue Jan 24 18:57:07 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jan 25 14:21:37 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++--
examples/test_case_data/sandbox/kylin.properties | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 07cb124..5932197 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -930,11 +930,11 @@ abstract public class KylinConfigBase implements Serializable {
//zoo keeper
public String getZooKeeperHost() {
- return getOptional("kylin.storage-zookeeper.host", "localhost");
+ return getOptional("kylin.storage.lock-manager-zk-host", "localhost");
}
public String getZooKeeperPort() {
- return getOptional("kylin.storage-zookeeper.port", "2181");
+ return getOptional("kylin.storage.lock-manager-zk-port", "2181");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index fa0c6bc..6cb5148 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -55,6 +55,9 @@ kylin.env.hdfs-working-dir=/kylin
kylin.engine.mr.reduce-input-mb=500
+#zoo keeper host in lock manager
+kylin.storage.lock-manager-zk-host=sandbox
+
### JOB ###
# max job retry on error, default 0: no retry
@@ -179,5 +182,3 @@ kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
-#zoo keeper
-kylin.storage-zookeeper.host=sandbox
\ No newline at end of file
[33/47] kylin git commit: Add Zookeeper Lock
Posted by bi...@apache.org.
Add Zookeeper Lock
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3276e2e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3276e2e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3276e2e
Branch: refs/heads/KYLIN-2361
Commit: d3276e2e909d3001724ee8fda1304ae8b7f08c63
Parents: d23bf93
Author: xiefan46 <95...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 22 +-
.../test_case_data/sandbox/kylin.properties | 4 +
.../storage/hdfs/ITHDFSResourceStoreTest.java | 66 +-----
.../kylin/storage/hdfs/ITLockManagerTest.java | 205 +++++++++++++++++++
.../kylin/storage/hbase/HBaseResourceStore.java | 1 +
.../org/apache/kylin/storage/hdfs/HDFSLock.java | 41 ----
.../kylin/storage/hdfs/HDFSLockManager.java | 45 ----
.../kylin/storage/hdfs/HDFSResourceStore.java | 95 +++++++--
.../apache/kylin/storage/hdfs/LockManager.java | 116 +++++++++++
.../apache/kylin/storage/hdfs/ResourceLock.java | 51 +++++
10 files changed, 471 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 44d636d..75b38ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,6 +193,14 @@ abstract public class KylinConfigBase implements Serializable {
return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
+ public String getRawHdfsWorkingDirectory() {
+ String root = getRequired("kylin.env.hdfs-working-dir");
+ if (!root.endsWith("/")) {
+ root += "/";
+ }
+ return root;
+ }
+
// ============================================================================
// METADATA
// ============================================================================
@@ -201,11 +209,6 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.metadata.url");
}
- //for hdfs resource store
- public String getHDFSMetadataUrl() {
- return getOptional("kylin.metadata.hdfs.url", "kylin_default_instance_hdfs@hdfs");
- }
-
// for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
@@ -925,4 +928,13 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+ //zoo keeper
+ public String getZooKeeperHost() {
+ return getOptional("kylin.storage-zookeeper.host", "localhost");
+ }
+
+ public String getZooKeeperPort() {
+ return getOptional("kylin.storage-zookeeper.port", "2181");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index b01c377..2c2da91 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -177,3 +177,7 @@ kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
#kylin.engine.spark-conf.spark.yarn.queue=default
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+
+#zoo keeper
+kylin.storage-zookeeper.host=sandbox
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ef04957..27d8a3c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -17,23 +17,15 @@
*/
package org.apache.kylin.storage.hdfs;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertEquals;
/**
* Created by xiefan on 17-1-10.
@@ -53,65 +45,13 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
this.cleanupTestMetadata();
}
- @Ignore
- @Test
- public void testHDFSUrl() throws Exception {
- assertEquals("kylin_default_instance_hdfs@hdfs", kylinConfig.getHDFSMetadataUrl());
- System.out.println("hdfs working dir : " + kylinConfig.getHdfsWorkingDirectory());
- }
-
- @Ignore
@Test
- public void testMultiThreadWriteHDFS() throws Exception{
- //System.out.println(kylinConfig.getHdfsWorkingDirectory());
- final Path testDir = new Path("hdfs:///test123");
- final FileSystem fs = HadoopUtil.getFileSystem(testDir);
- final String fileName = "test.json";
- int threadNum = 3;
- ExecutorService service = Executors.newFixedThreadPool(threadNum);
- final CountDownLatch latch = new CountDownLatch(threadNum);
- Path p = new Path(testDir,fileName);
- fs.deleteOnExit(p);
- fs.createNewFile(p);
- for(int i=0;i<threadNum;i++) {
- service.execute(new Runnable() {
- @Override
- public void run() {
- try {
- long id = Thread.currentThread().getId();
- Path p = new Path(testDir, fileName);
- /*while(fs.exists(p)){
- System.out.println("Thread id : " + id + " can not get lock,sleep a while");
- Thread.currentThread().sleep(1000);
- }*/
- while(!fs.createNewFile(p)){
- System.out.println("Thread id : " + id + " can not get lock,sleep a while");
- Thread.currentThread().sleep(1000);
- }
- System.out.println("Thread id : " + id + " get lock, sleep a while");
- Thread.currentThread().sleep(1000);
- fs.delete(p,true);
- System.out.println("Thread id : " + id + " release lock");
- latch.countDown();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
- Thread.currentThread().sleep(1000);
- fs.delete(p,true);
- System.out.println("main thread release lock.Waiting threads down");
- System.out.println("file still exist : " + fs.exists(p));
- latch.await();
- }
-
- @Test
- public void testHDFSStore() throws Exception {
+ public void testResourceStoreBasic() throws Exception {
KylinConfig config = KylinConfig.getInstanceFromEnv();
ResourceStore store = new HDFSResourceStore(config);
ResourceStoreTest.testAStore(store);
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
new file mode 100644
index 0000000..2b58d30
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.storage.hdfs;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.utils.CloseableUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertEquals;
+
+public class ITLockManagerTest extends HBaseMetadataTestCase {
+
+
+ private String zkConnection = "sandbox:2181";
+
+ private KylinConfig kylinConfig;
+
+ private CuratorFramework zkClient;
+
+ private static final String lockRootPath = "/test_lock";
+
+ private LockManager manager;
+
+ private static final int QTY = 5;
+
+ private static final int REPETITIONS = QTY * 10;
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy);
+ zkClient.start();
+ manager = new LockManager(kylinConfig, lockRootPath);
+ System.out.println("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
+
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ zkClient.delete().deletingChildrenIfNeeded().forPath(lockRootPath);
+ List<String> nodes = zkClient.getChildren().forPath("/");
+ System.out.println("nodes in zk after delete : " + nodes);
+ manager.close();
+ }
+
+ @Test
+ public void testCreateLock() throws Exception {
+
+ ResourceLock lock = manager.getLock("/dictionary/numberdict.json");
+ lock.acquire();
+ manager.releaseLock(lock);
+ System.out.println(zkClient.getChildren().forPath(lockRootPath + "/dictionary"));
+ List<String> nodes = zkClient.getChildren().forPath(lockRootPath + "/dictionary");
+ assertEquals(1, nodes.size());
+ assertEquals("numberdict.json", nodes.get(0));
+ }
+
+ @Test
+ public void testLockSafty() throws Exception {
+ // all of the useful sample code is in ExampleClientThatLocks.java
+
+ // FakeLimitedResource simulates some external resource that can only be access by one process at a time
+ final FakeLimitedResource resource = new FakeLimitedResource();
+ ExecutorService service = Executors.newFixedThreadPool(QTY);
+ final TestingServer server = new TestingServer(zkConnection);
+ final List<FutureTask<Void>> tasks = new ArrayList<>();
+ try {
+ for (int i = 0; i < QTY; ++i) {
+ final int index = i;
+ FutureTask<Void> task = new FutureTask<Void>(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ LockManager threadLocalLockManager = new LockManager(kylinConfig, lockRootPath);
+ try {
+ ExampleClientThatLocks example = new ExampleClientThatLocks(threadLocalLockManager, lockRootPath, resource, "Client " + index);
+ for (int j = 0; j < REPETITIONS; ++j) {
+ example.doWork(10, TimeUnit.SECONDS);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ e.printStackTrace();
+ // log or do something
+ } finally {
+ threadLocalLockManager.close();
+ }
+ return null;
+ }
+ });
+ tasks.add(task);
+ service.submit(task);
+ }
+ for (FutureTask<Void> task : tasks) {
+ task.get();
+ }
+ } finally {
+ CloseableUtils.closeQuietly(server);
+ }
+ }
+
+ class FakeLimitedResource {
+ private final AtomicBoolean inUse = new AtomicBoolean(false);
+
+ public void use() throws InterruptedException {
+ // in a real application this would be accessing/manipulating a shared resource
+
+ if (!inUse.compareAndSet(false, true)) {
+ throw new IllegalStateException("Needs to be used by one client at a time");
+ }
+
+ try {
+ Thread.sleep((long) (3 * Math.random()));
+ } finally {
+ inUse.set(false);
+ }
+ }
+ }
+
+ class TestingServer implements Closeable {
+
+ private String connectionString;
+
+ public TestingServer(String connectionStr) {
+ this.connectionString = connectionStr;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ public String getConnectString() {
+ return connectionString;
+ }
+ }
+
+ class ExampleClientThatLocks {
+
+ private final FakeLimitedResource resource;
+
+ private final String clientName;
+
+ private LockManager lockManager;
+
+ private String lockPath;
+
+ public ExampleClientThatLocks(LockManager lockManager, String lockPath, FakeLimitedResource resource, String clientName) {
+ this.resource = resource;
+ this.clientName = clientName;
+ this.lockManager = lockManager;
+ this.lockPath = lockPath;
+ }
+
+ public void doWork(long time, TimeUnit unit) throws Exception {
+ ResourceLock lock = lockManager.getLock(lockPath);
+ if (!lock.acquire(time, unit)) {
+ throw new IllegalStateException(clientName + " could not acquire the lock");
+ }
+ try {
+ System.out.println(clientName + " has the lock");
+ resource.use();
+ } finally {
+ System.out.println(clientName + " releasing the lock");
+ lock.release(); // always release the lock in a finally block
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1c45967..170e351 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -311,6 +311,7 @@ public class HBaseResourceStore extends ResourceStore {
} finally {
IOUtils.closeQuietly(table);
}
+
}
private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
deleted file mode 100644
index 8710edf..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLock.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.apache.kylin.storage.hdfs;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * Created by xiefan on 17-1-17.
- */
-public class HDFSLock {
-
- private Path rawLock;
-
- private static final Logger logger = LoggerFactory.getLogger(HDFSLock.class);
-
- protected HDFSLock(String resourceFullPath) {
- this.rawLock = new Path(resourceFullPath);
- }
-
- public boolean init(FileSystem fs) throws IOException, InterruptedException {
- if (!fs.isFile(rawLock)) {
- logger.info("Not support directory lock yet");
- return false;
- }
- while (!fs.createNewFile(rawLock)) {
- Thread.currentThread().sleep(1000);
- }
- return true;
- }
-
- public boolean release(FileSystem fs) throws IOException, InterruptedException {
- while (!fs.delete(rawLock, false)) {
- Thread.currentThread().sleep(1000);
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
deleted file mode 100644
index 1cd0800..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSLockManager.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.kylin.storage.hdfs;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.engine.mr.HadoopUtil;
-
-import java.io.IOException;
-
-/**
- * Created by xiefan on 17-1-17.
- */
-public class HDFSLockManager {
-
- private static final String LOCK_HOME = "LOCK_HOME";
-
- private Path lockPath;
-
- private FileSystem fs;
-
- public HDFSLockManager(String hdfsWorkingDir) throws IOException{
- this.lockPath = new Path(hdfsWorkingDir,LOCK_HOME);
- this.fs = HadoopUtil.getFileSystem(lockPath);
- if(!fs.exists(lockPath)){
- fs.create(lockPath);
- }
- }
-
- public HDFSLock getLock(String resourceFullPath) throws IOException,InterruptedException,IllegalStateException{
- HDFSLock lock = new HDFSLock(resourceFullPath);
- boolean success = lock.init(fs);
- if(success){
- return lock;
- }else{
- throw new IllegalStateException("Try get lock fail. Resourse path : " + resourceFullPath);
- }
- }
-
- public void releaseLock(HDFSLock lock) throws IOException,InterruptedException,IllegalStateException{
- boolean success = lock.release(fs);
- if(!success)
- throw new IllegalStateException("Release lock fail");
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index c7f0f25..a746a97 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,44 +39,62 @@ import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
-/**
- * Created by xiefan on 17-1-10.
- */
public class HDFSResourceStore extends ResourceStore {
- private static final String DEFAULT_TABLE_NAME = "kylin_default_instance_hdfs";
+ private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
+
+ private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 10;
+
+ private static final String DEFAULT_FOLDER_NAME = "kylin_default_instance";
+
+ private static final String DEFAULT_METADATA_FOLDER_NAME = "hdfs_metadata";
private Path hdfsMetaPath;
private FileSystem fs;
- private HDFSLockManager lockManager;
-
- private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
+ private LockManager lockManager;
//public for test. Normal should be protected
- public HDFSResourceStore(KylinConfig kylinConfig) throws IOException {
+ public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
super(kylinConfig);
- String metadataUrl = kylinConfig.getHDFSMetadataUrl();
- // split TABLE@HBASE_URL
+ String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
- String metaDirName = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+ String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut);
+ metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
+ logger.info("meta dir name :" + metaDirName);
createMetaFolder(metaDirName, kylinConfig);
}
- private void createMetaFolder(String metaDirName, KylinConfig kylinConfig) throws IOException {
+ private void createMetaFolder(String metaDirName, KylinConfig kylinConfig) throws Exception {
String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
fs = HadoopUtil.getFileSystem(hdfsWorkingDir);
+ logger.info("hdfs working dir : " + hdfsWorkingDir);
Path hdfsWorkingPath = new Path(hdfsWorkingDir);
if (!fs.exists(hdfsWorkingPath)) {
throw new IOException("HDFS working dir not exist");
}
+ //creat lock manager
+ this.lockManager = new LockManager(kylinConfig, kylinConfig.getRawHdfsWorkingDirectory() + metaDirName);
+ //create hdfs meta path
hdfsMetaPath = new Path(hdfsWorkingPath, metaDirName);
if (!fs.exists(hdfsMetaPath)) {
- fs.create(hdfsMetaPath, true);
+ ResourceLock lock = lockManager.getLock(lockManager.getLockPath("/"));
+ try {
+ if (lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES)) {
+ logger.info("get root lock successfully");
+ if (!fs.exists(hdfsMetaPath)) {
+ fs.mkdirs(hdfsMetaPath);
+ logger.info("create hdfs meta path");
+ }
+ }
+ } finally {
+ lockManager.releaseLock(lock);
+ }
}
- lockManager = new HDFSLockManager(hdfsWorkingDir);
+ logger.info("hdfs meta path : " + hdfsMetaPath.toString());
}
@Override
@@ -132,7 +150,8 @@ public class HDFSResourceStore extends ResourceStore {
logger.warn("Zero length file: " + p.toString());
}
FSDataInputStream in = fs.open(p);
- return new RawResource(fs.open(p), getResourceTimestamp(resPath));
+ long t = in.readLong();
+ return new RawResource(in, t);
} else {
return null;
}
@@ -144,19 +163,42 @@ public class HDFSResourceStore extends ResourceStore {
if (!fs.exists(p) || !fs.isFile(p)) {
return 0;
}
- FileStatus status = fs.getFileStatus(p);
- return status.getModificationTime();
+ FSDataInputStream in = null;
+ ResourceLock lock = null;
+ try {
+ lock = lockManager.getLock(resPath);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
+ in = fs.open(p);
+ long t = in.readLong();
+ return t;
+ } catch (Exception e) {
+ throw new IOException("Put resource fail", e);
+ } finally {
+ IOUtils.closeQuietly(in);
+ lockManager.releaseLock(lock);
+ }
+
}
@Override
protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+ logger.info("res path : " + resPath);
Path p = getRealHDFSPath(resPath);
+ logger.info("put resource : " + p.toUri());
FSDataOutputStream out = null;
+ ResourceLock lock = null;
try {
+ lock = lockManager.getLock(resPath);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
out = fs.create(p, true);
+ out.writeLong(ts);
IOUtils.copy(content, out);
+
+ } catch (Exception e) {
+ throw new IOException("Put resource fail", e);
} finally {
IOUtils.closeQuietly(out);
+ lockManager.releaseLock(lock);
}
}
@@ -180,9 +222,18 @@ public class HDFSResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- Path p = getRealHDFSPath(resPath);
- if (fs.exists(p)) {
- fs.delete(p, true);
+ ResourceLock lock = null;
+ try {
+ lock = lockManager.getLock(resPath);
+ lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES);
+ Path p = getRealHDFSPath(resPath);
+ if (fs.exists(p)) {
+ fs.delete(p, true);
+ }
+ } catch (Exception e) {
+ throw new IOException("Delete resource fail", e);
+ } finally {
+ lockManager.releaseLock(lock);
}
}
@@ -192,6 +243,8 @@ public class HDFSResourceStore extends ResourceStore {
}
private Path getRealHDFSPath(String resourcePath) {
+ if (resourcePath.startsWith("/") && resourcePath.length() > 1)
+ resourcePath = resourcePath.substring(1, resourcePath.length());
return new Path(this.hdfsMetaPath, resourcePath);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
new file mode 100644
index 0000000..4959718
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.storage.hdfs;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+
+public class LockManager {
+
+ private static Logger logger = LoggerFactory.getLogger(ZookeeperDistributedJobLock.class);
+
+ final private KylinConfig config;
+
+ final CuratorFramework zkClient;
+
+ private String lockRootPath;
+
+ public LockManager(String lockRootPath) throws Exception {
+
+ this(KylinConfig.getInstanceFromEnv(), lockRootPath);
+ }
+
+ public LockManager(KylinConfig config, String lockRootPath) throws Exception {
+ this.config = config;
+ this.lockRootPath = lockRootPath;
+ String zkConnectString = getZKConnectString(config);
+ logger.info("zk connection string:" + zkConnectString);
+ if (StringUtils.isEmpty(zkConnectString)) {
+ throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
+ }
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ zkClient = CuratorFrameworkFactory.newClient(zkConnectString, retryPolicy);
+ zkClient.start();
+ if (zkClient.checkExists().forPath(lockRootPath) == null)
+ zkClient.create().creatingParentsIfNeeded().forPath(lockRootPath);
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ close();
+ }
+ }));
+ }
+
+ public ResourceLock getLock(String name) throws Exception {
+ String lockPath = getLockPath(name);
+ InterProcessMutex lock = new InterProcessMutex(zkClient, lockPath);
+ return new ResourceLock(lockPath, lock);
+ }
+
+ public void releaseLock(ResourceLock lock) {
+ try {
+ if (lock != null)
+ lock.release();
+ } catch (Exception e) {
+ logger.error("Fail to release lock");
+ e.printStackTrace();
+ }
+ }
+
+ private static String getZKConnectString(KylinConfig kylinConfig) {
+ final String host = kylinConfig.getZooKeeperHost();
+ final String port = kylinConfig.getZooKeeperPort();
+ return StringUtils.join(Iterables.transform(Arrays.asList(host.split(",")), new Function<String, String>() {
+ @Nullable
+ @Override
+ public String apply(String input) {
+ return input + ":" + port;
+ }
+ }), ",");
+ }
+
+ public String getLockPath(String resourceName) {
+ if (!resourceName.startsWith("/"))
+ resourceName = "/" + resourceName;
+ if (resourceName.endsWith("/"))
+ resourceName = resourceName.substring(0, resourceName.length() - 1);
+ return lockRootPath + resourceName;
+ }
+
+ public void close() {
+ try {
+ zkClient.close();
+ } catch (Exception e) {
+ logger.error("error occurred to close PathChildrenCache", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
new file mode 100644
index 0000000..9d51871
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.storage.hdfs;
+
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+
+import java.util.concurrent.TimeUnit;
+
+
+public class ResourceLock {
+
+ private String resourcePath;
+
+ private InterProcessMutex lock;
+
+ protected ResourceLock(String resourcePath, InterProcessMutex lock) {
+ this.resourcePath = resourcePath;
+ this.lock = lock;
+ }
+
+ public boolean acquire(long time, TimeUnit unit) throws Exception {
+ return lock.acquire(time, unit);
+ }
+
+ public void acquire() throws Exception{
+ lock.acquire();
+ }
+
+ protected void release() throws Exception {
+ lock.release();
+ }
+
+ public String getResourcePath() {
+ return resourcePath;
+ }
+}
[06/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and
Avatica to 1.9.0
Posted by bi...@apache.org.
KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9
Branch: refs/heads/KYLIN-2361
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu <bi...@apache.org>
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 19 14:54:20 2017 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 11 +-
.../calcite/sql2rel/SqlToRelConverter.java | 3941 +++++++++++-------
jdbc/pom.xml | 12 +-
.../java/org/apache/kylin/jdbc/KylinMeta.java | 2 +-
kylin-it/pom.xml | 14 +-
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 1 +
pom.xml | 10 +-
query/pom.xml | 10 +-
8 files changed, 2430 insertions(+), 1571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica</artifactId>
</dependency>
</dependencies>
-
</project>
[08/47] kylin git commit: KYLIN-2409 change inmem cubing to single
thread by default
Posted by bi...@apache.org.
KYLIN-2409 change inmem cubing to single thread by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21969753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21969753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21969753
Branch: refs/heads/KYLIN-2361
Commit: 21969753cc66efa22bc9fc933af46346e9846631
Parents: b49c9e3
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 19 11:52:17 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 19 17:37:42 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 6 +++++-
.../kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java | 2 +-
.../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 9 +++++++++
.../org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java | 2 ++
4 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d6774ff..74903d5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -300,6 +300,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
}
+ public int getCubeAlgorithmInMemConcurrentThreads() {
+ return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-concurrent-threads", "1"));
+ }
+
public boolean isIgnoreCubeSignatureInconsistency() {
return Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", "false"));
}
@@ -744,7 +748,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getYarnStatusCheckIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60"));
+ return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "10"));
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 651203a..c7a4a05 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -46,7 +46,7 @@ abstract public class AbstractInMemCubeBuilder {
final protected CubeDesc cubeDesc;
final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
- protected int taskThreadCount = 4;
+ protected int taskThreadCount = 1;
protected int reserveMemoryMB = 100;
public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 55c04c6..9158aa3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -178,11 +179,16 @@ public class GTAggregateScanner implements IGTScanner {
final List<Dump> dumps;
final int keyLength;
final boolean[] compareMask;
+ boolean compareAll = true;
final BufferedMeasureCodec measureCodec;
final Comparator<byte[]> bytesComparator = new Comparator<byte[]>() {
@Override
public int compare(byte[] o1, byte[] o2) {
+ if (compareAll) {
+ return Bytes.compareTo(o1, o2);
+ }
+
int result = 0;
// profiler shows this check is slow
// Preconditions.checkArgument(keyLength == o1.length && keyLength == o2.length);
@@ -206,6 +212,9 @@ public class GTAggregateScanner implements IGTScanner {
public AggregationCache() {
compareMask = createCompareMask();
+ for (boolean l : compareMask) {
+ compareAll = compareAll && l;
+ }
keyLength = compareMask.length;
dumps = Lists.newArrayList();
aggBufMap = createBuffMap();
http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index 116d5e0..c0ff2f2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -94,8 +94,10 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
dictionaryMap.put(col, cubeSegment.getDictionary(col));
}
+ int taskCount = config.getCubeAlgorithmInMemConcurrentThreads();
DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
cubeBuilder.setReserveMemoryMB(calculateReserveMB(context.getConfiguration()));
+ cubeBuilder.setConcurrentThreads(taskCount);
ExecutorService executorService = Executors.newSingleThreadExecutor();
future = executorService.submit(cubeBuilder.buildAsRunnable(queue, new MapContextGTRecordWriter(context, cubeDesc, cubeSegment)));
[31/47] kylin git commit: KYLIN-2242 write multiple files in
FactDistinctColumnsReducer with MultipleOutputs
Posted by bi...@apache.org.
KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6dda4fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6dda4fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6dda4fe
Branch: refs/heads/KYLIN-2361
Commit: f6dda4fecd5a882a17379987445a5aa0a347ff63
Parents: 4c33e38
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HadoopUtil.java | 16 ++
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../kylin/engine/mr/common/BatchConstants.java | 9 +-
.../engine/mr/steps/CreateDictionaryJob.java | 43 ++--
.../engine/mr/steps/FactDistinctColumnsJob.java | 32 ++-
.../mr/steps/FactDistinctColumnsReducer.java | 240 +++++++------------
.../engine/mr/steps/SaveStatisticsStep.java | 10 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 10 +-
8 files changed, 175 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
}
}
+ public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final String filter) throws IOException {
+ FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(filter);
+ }
+ });
+
+ if (fileStatus.length == 1) {
+ return fileStatus[0].getPath();
+ } else {
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
}
public String getStatisticsPath(String jobId) {
- return getRealizationRootPath(jobId) + "/statistics";
+ return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + BatchConstants.CFG_OUTPUT_STATISTICS;
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
String CFG_STATISTICS_ENABLED = "statistics.enabled";
String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for compatibility issue better not change it
String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
- String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
+ String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+ String CFG_OUTPUT_COLUMN = "column";
+ String CFG_OUTPUT_DICT = "dict";
+ String CFG_OUTPUT_STATISTICS = "statistics";
+ String CFG_OUTPUT_PARTITION = "partition";
+
+
/**
* command line ARGuments
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 95d8cb1..e5d053b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -18,15 +18,20 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.DataInputStream;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.commons.cli.Options;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.HadoopUtil;
@@ -63,21 +68,27 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getWorkingFileSystem();
- if (fs.exists(dictFile) == false)
+ Path colDir = new Path(factColumnsInputPath, col.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+
+ Path dictFile = HadoopUtil.getFilterOnlyPath(fs, colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ if (dictFile == null) {
return null;
-
- FSDataInputStream is = null;
- try {
- is = fs.open(dictFile);
- String dictClassName = is.readUTF();
- Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
- dict.readFields(is);
- logger.info("DictionaryProvider read dict from file: " + dictFile);
- return dict;
- } finally {
- IOUtils.closeQuietly(is);
+ }
+
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(HadoopUtil.getCurrentConfiguration(), SequenceFile.Reader.file(dictFile))) {
+ NullWritable key = NullWritable.get();
+ BytesWritable value = new BytesWritable();
+ reader.next(key, value);
+
+ ByteBuffer buffer = new ByteArray(value.getBytes()).asBuffer();
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(buffer))) {
+ String dictClassName = is.readUTF();
+ Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
+ dict.readFields(is);
+ logger.info("DictionaryProvider read dict from file: " + dictFile);
+ return dict;
+ }
}
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index ce01eb6..aded600 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -23,11 +23,16 @@ import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -82,8 +87,8 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
int uhcReducerCount = cube.getConfig().getUHCReducerCount();
int[] uhcIndex = cubeMgr.getUHCIndex(cube.getDescriptor());
- for(int index : uhcIndex) {
- if(index == 1) {
+ for (int index : uhcIndex) {
+ if (index == 1) {
reducerCount += uhcReducerCount - 1;
}
}
@@ -92,7 +97,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
throw new IllegalArgumentException("The max reducer number for FactDistinctColumnsJob is 255, but now it is " + reducerCount + ", decrease 'kylin.engine.mr.uhc-reducer-count'");
}
-
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled);
@@ -117,6 +121,12 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
attachCubeMetadata(cube, job.getConfiguration());
+ /**
+ * don't compress the reducer output so that {@link CreateDictionaryJob} and {@link UpdateCubeInfoAfterBuildStep}
+ * could read the reducer file directly
+ */
+ job.getConfiguration().set(BatchConstants.CFG_MAPRED_OUTPUT_COMPRESS, "false");
+
return waitForCompletion(job);
} finally {
@@ -138,18 +148,22 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
private void setupReducer(Path output, int numberOfReducers) throws IOException {
job.setReducerClass(FactDistinctColumnsReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Text.class);
job.setPartitionerClass(FactDistinctColumnPartitioner.class);
job.setNumReduceTasks(numberOfReducers);
- // important, reducer writes HDFS directly at the moment
- job.setReduceSpeculativeExecution(false);
-
+ //make each reducer output to respective dir
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_COLUMN, SequenceFileOutputFormat.class, NullWritable.class, Text.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_DICT, SequenceFileOutputFormat.class, NullWritable.class, BytesWritable.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_STATISTICS, SequenceFileOutputFormat.class, LongWritable.class, BytesWritable.class);
+ MultipleOutputs.addNamedOutput(job, BatchConstants.CFG_OUTPUT_PARTITION, TextOutputFormat.class, NullWritable.class, LongWritable.class);
+
+
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
+ //prevent to create zero-sized default output
+ LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
+
deletePath(job.getConfiguration(), output);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 711d991..5d2fb72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -18,27 +18,25 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -47,7 +45,7 @@ import org.apache.kylin.dict.IDictionaryBuilder;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CubeStatsWriter;
+import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -63,14 +61,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
private List<TblColRef> columnList;
- private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
protected Map<Long, HLLCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
private int samplingPercentage;
- private List<ByteArray> colValues;
private TblColRef col = null;
private boolean isStatistics = false;
private KylinConfig cubeConfig;
@@ -88,10 +84,14 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
public static final String DICT_FILE_POSTFIX = ".rldict";
public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
+ private MultipleOutputs mos;
+
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
+ mos = new MultipleOutputs(context);
+
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
@@ -109,26 +109,20 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (collectStatistics && (taskId == numberOfTasks - 1)) {
// hll
isStatistics = true;
- statisticsOutput = conf.get(BatchConstants.CFG_STATISTICS_OUTPUT);
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
logger.info("Reducer " + taskId + " handling stats");
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
- isStatistics = false;
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
logger.info("Do not have partition col. This reducer will keep empty");
}
- colValues = Lists.newLinkedList();
- logger.info("Reducer " + taskId + " handling partition column " + col);
} else {
// normal col
- isStatistics = false;
col = columnList.get(reducerIdToColumnIndex.get(taskId));
- colValues = Lists.newLinkedList();
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
@@ -194,15 +188,13 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
logAFewRows(value);
builder.addValue(value);
} else {
- colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
- if (colValues.size() == 1000000) { //spill every 1 million
- logger.info("spill values to disk...");
- outputDistinctValues(col, colValues, context);
- colValues.clear();
- }
+ byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
+ // output written to baseDir/colName/-r-00000 (etc)
+ String fileName = col.getName() + "/";
+ mos.write(BatchConstants.CFG_OUTPUT_COLUMN, NullWritable.get(), new Text(keyBytes), fileName);
}
}
-
+
rowCount++;
}
@@ -212,162 +204,104 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
- private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
- final Configuration conf = context.getConfiguration();
- final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getIdentity());
- final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
- final Path outputFile = new Path(colDir, fileName);
-
- FSDataOutputStream out = null;
- try {
- if (!fs.exists(colDir)) {
- fs.mkdirs(colDir);
- }
-
- if (fs.exists(outputFile)) {
- out = fs.append(outputFile);
- logger.info("append file " + outputFile);
- } else {
- out = fs.create(outputFile);
- logger.info("create file " + outputFile);
- }
-
- for (ByteArray value : values) {
- out.write(value.array(), value.offset(), value.length());
- out.write('\n');
- }
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
- final String fileName = col.getIdentity() + DICT_FILE_POSTFIX;
- FSDataOutputStream out = getOutputStream(context, fileName);
- try {
- String dictClassName = dict.getClass().getName();
- out.writeUTF(dictClassName);
- dict.write(out);
- logger.info("reducer id is:+" + taskId + " colName:" + col.getName() + " writing dict at file : " + fileName + " dict class:" + dictClassName);
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private void outputPartitionInfo(Context context) throws IOException {
- final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX;
- FSDataOutputStream out = getOutputStream(context, fileName);
- try {
- out.writeLong(timeMinValue);
- out.writeLong(timeMaxValue);
- logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
- } finally {
- IOUtils.closeQuietly(out);
- }
- }
-
- private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
- final Configuration conf = context.getConfiguration();
- final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
- final Path outputFile = new Path(outputPath, outputFileName);
- if (!fs.exists(outputPath)) {
- fs.mkdirs(outputPath);
- }
- FSDataOutputStream out = fs.create(outputFile);
- return out;
- }
-
@Override
protected void doCleanup(Context context) throws IOException, InterruptedException {
if (isStatistics) {
- // output the hll info
- long grandTotal = 0;
- for (HLLCounter hll : cuboidHLLMap.values()) {
- grandTotal += hll.getCountEstimate();
- }
- double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
-
- int mapperNumber = baseCuboidRowCountInMappers.size();
+ //output the hll info;
+ List<Long> allCuboids = Lists.newArrayList();
+ allCuboids.addAll(cuboidHLLMap.keySet());
+ Collections.sort(allCuboids);
- writeMapperAndCuboidStatistics(context); // for human check
- CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
- cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
+ logMapperAndCuboidStatistics(allCuboids); // for human check
+ outputStatistics(allCuboids);
} else if (isPartitionCol) {
// partition col
- if (col != null) {
- outputPartitionInfo(context);
- }
+ outputPartitionInfo();
} else {
// normal col
if (isReducerLocalBuildDict) {
Dictionary<String> dict = builder.build();
- outputDict(col, dict, context);
- } else {
- if (colValues.size() > 0) {
- outputDistinctValues(col, colValues, context);
- colValues.clear();
- }
+ outputDict(col, dict);
}
}
+
+ mos.close();
}
- private void writeMapperAndCuboidStatistics(Context context) throws IOException {
- Configuration conf = context.getConfiguration();
- FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
- Path path = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME);
- FSDataOutputStream out = fs.create(path);
+ private void outputPartitionInfo() throws IOException, InterruptedException {
+ if (col != null) {
+ // output written to baseDir/colName/colName.pci-r-00000 (etc)
+ String partitionFileName = col.getName() + "/" + col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
- try {
- String msg;
+ mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMinValue), partitionFileName);
+ mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new LongWritable(timeMaxValue), partitionFileName);
+ logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
+ }
+ }
- List<Long> allCuboids = Lists.newArrayList();
- allCuboids.addAll(cuboidHLLMap.keySet());
- Collections.sort(allCuboids);
+ private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
+ // output written to baseDir/colName/colName.rldict-r-00000 (etc)
+ String dictFileName = col.getName() + "/" + col.getName() + DICT_FILE_POSTFIX;
- msg = "Total cuboid number: \t" + allCuboids.size();
- writeLine(out, msg);
- msg = "Samping percentage: \t" + samplingPercentage;
- writeLine(out, msg);
-
- writeLine(out, "The following statistics are collected based on sampling data.");
- writeLine(out, "Number of Mappers: " + baseCuboidRowCountInMappers.size());
- for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
- if (baseCuboidRowCountInMappers.get(i) > 0) {
- msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
- writeLine(out, msg);
- }
- }
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) {
+ outputStream.writeUTF(dict.getClass().getName());
+ dict.write(outputStream);
- long grantTotal = 0;
- for (long i : allCuboids) {
- grantTotal += cuboidHLLMap.get(i).getCountEstimate();
- msg = "Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate();
- writeLine(out, msg);
- }
+ mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new BytesWritable(baos.toByteArray()), dictFileName);
+ }
+ }
- msg = "Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge;
- writeLine(out, msg);
+ private void outputStatistics(List<Long> allCuboids) throws IOException, InterruptedException {
+ // output written to baseDir/statistics/statistics-r-00000 (etc)
+ String statisticsFileName = BatchConstants.CFG_OUTPUT_STATISTICS + "/" + BatchConstants.CFG_OUTPUT_STATISTICS;
- msg = "After merge, the cube has row count: \t " + grantTotal;
- writeLine(out, msg);
+ ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
- if (grantTotal > 0) {
- msg = "The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal;
- writeLine(out, msg);
- }
+ // mapper overlap ratio at key -1
+ long grandTotal = 0;
+ for (HLLCounter hll : cuboidHLLMap.values()) {
+ grandTotal += hll.getCountEstimate();
+ }
+ double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-1), new BytesWritable(Bytes.toBytes(mapperOverlapRatio)), statisticsFileName);
+
+ // mapper number at key -2
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-2), new BytesWritable(Bytes.toBytes(baseCuboidRowCountInMappers.size())), statisticsFileName);
+
+ // sampling percentage at key 0
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(0L), new BytesWritable(Bytes.toBytes(samplingPercentage)), statisticsFileName);
- } finally {
- IOUtils.closeQuietly(out);
+ for (long i : allCuboids) {
+ valueBuf.clear();
+ cuboidHLLMap.get(i).writeRegisters(valueBuf);
+ valueBuf.flip();
+ mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()), statisticsFileName);
}
}
- private void writeLine(FSDataOutputStream out, String msg) throws IOException {
- out.write(msg.getBytes());
- out.write('\n');
+ private void logMapperAndCuboidStatistics(List<Long> allCuboids) throws IOException {
+ logger.info("Total cuboid number: \t" + allCuboids.size());
+ logger.info("Samping percentage: \t" + samplingPercentage);
+ logger.info("The following statistics are collected based on sampling data.");
+ logger.info("Number of Mappers: " + baseCuboidRowCountInMappers.size());
+ for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
+ if (baseCuboidRowCountInMappers.get(i) > 0) {
+ logger.info("Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i));
+ }
+ }
+
+ long grantTotal = 0;
+ for (long i : allCuboids) {
+ grantTotal += cuboidHLLMap.get(i).getCountEstimate();
+ logger.info("Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate());
+ }
+
+ logger.info("Sum of all the cube segments (before merge) is: \t " + totalRowsBeforeMerge);
+ logger.info("After merge, the cube has row count: \t " + grantTotal);
+ if (grantTotal > 0) {
+ logger.info("The mapper overlap ratio is: \t" + totalRowsBeforeMerge / grantTotal);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 2671042..28f99fb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -60,9 +60,11 @@ public class SaveStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
FileSystem fs = HadoopUtil.getWorkingFileSystem();
- Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- if (!fs.exists(statisticsFilePath))
- throw new IOException("File " + statisticsFilePath + " does not exists");
+ Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
+ Path statisticsFilePath = HadoopUtil.getFilterOnlyPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS);
+ if (statisticsFilePath == null) {
+ throw new IOException("fail to find the statistics file in base dir: " + statisticsDir);
+ }
FSDataInputStream is = fs.open(statisticsFilePath);
try {
@@ -110,7 +112,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
+
// in-mem cubing is good when
// 1) the cluster has enough mapper slots to run in parallel
// 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dc80399..81d5c42 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -80,8 +80,13 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
+ Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+ Path outputFile = HadoopUtil.getFilterOnlyPath(fs, colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ if (outputFile == null) {
+ throw new IOException("fail to find the partition file in base dir: " + colDir);
+ }
+
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
@@ -97,5 +102,4 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
segment.setDateRangeStart(minValue);
segment.setDateRangeEnd(maxValue);
}
-
}