You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/05/24 06:09:54 UTC
[1/8] kylin git commit: KYLIN-1708 GUI friendly tips for terms in
kylin [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/1.5.x-HBase1.x d97b525f2 -> fb740ec88 (forced update)
KYLIN-1708 GUI friendly tips for terms in kylin
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f0ed10e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f0ed10e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f0ed10e
Branch: refs/heads/1.5.x-HBase1.x
Commit: 5f0ed10e603ad8b5c1c88e44ce174b3a62ee5592
Parents: 682a2f1
Author: Jason <ji...@163.com>
Authored: Thu May 19 11:18:31 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu May 19 11:18:31 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 7 +
webapp/app/js/controllers/modelEdit.js | 12 +
webapp/app/js/controllers/modelSchema.js | 21 +-
webapp/app/js/directives/directives.js | 23 ++
webapp/app/js/model/cubeConfig.js | 3 +-
webapp/app/less/app.less | 33 +++
.../cubeDesigner/advanced_settings.html | 198 +++++++++-------
webapp/app/partials/cubeDesigner/info.html | 52 ++--
webapp/app/partials/cubeDesigner/measures.html | 40 ++--
.../partials/cubeDesigner/refresh_settings.html | 28 ++-
.../modelDesigner/conditions_settings.html | 237 +++++++++----------
.../app/partials/modelDesigner/model_info.html | 31 +--
12 files changed, 396 insertions(+), 289 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index b1aa08a..b6fc875 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -666,6 +666,13 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
});
+ $scope.removeNotificationEvents = function(){
+ if($scope.cubeMetaFrame.status_need_notify.indexOf('ERROR') == -1){
+ $scope.cubeMetaFrame.status_need_notify.unshift('ERROR');
+ }
+ }
+
+
$scope.$on('DimensionsEdited', function (event) {
if ($scope.cubeMetaFrame) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 903b584..a916aad 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -77,6 +77,9 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
project:$scope.projectModel.selectedProject
};
+ $scope.partitionColumn ={
+ "hasSeparateTimeColumn" : false
+ }
// ~ init
if ($scope.isEdit = !!$routeParams.modelName) {
@@ -84,6 +87,9 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
ModelDescService.query({model_name: modelName}, function (model) {
if (model) {
modelsManager.selectedModel = model;
+ if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
+ $scope.partitionColumn.hasSeparateTimeColumn = true;
+ }
modelsManager.selectedModel.project = ProjectModel.getProjectByCubeModel(modelName);
if(!ProjectModel.getSelectedProject()){
@@ -101,6 +107,12 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
}
+ $scope.toggleHasSeparateColumn = function(){
+ if($scope.partitionColumn.hasSeparateTimeColumn == false){
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_column = null;
+ }
+ }
+
$scope.prepareModel = function () {
// generate column family
$scope.state.project = modelsManager.selectedModel.project;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/js/controllers/modelSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelSchema.js b/webapp/app/js/controllers/modelSchema.js
index 7885758..e309873 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -28,6 +28,22 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
$scope.forms = {};
+ // ~ init
+ if (!$scope.state) {
+ $scope.state = {mode: "view"};
+ }
+
+ if(!$scope.partitionColumn){
+ $scope.partitionColumn ={
+ "hasSeparateTimeColumn" : false
+ }
+ }
+
+ if($scope.state.mode !== "edit" && $scope.modelsManager.selectedModel.partition_desc.partition_time_column){
+ $scope.partitionColumn.hasSeparateTimeColumn = true;
+ }
+
+
$scope.wizardSteps = [
{title: 'Model Info', src: 'partials/modelDesigner/model_info.html', isComplete: false, form: 'model_info_form'},
@@ -54,11 +70,6 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
$scope.curStep = $scope.wizardSteps[0];
-
- // ~ init
- if (!$scope.state) {
- $scope.state = {mode: "view"};
- }
//init modelsManager
if ($scope.state.mode == "edit") {
var defer = $q.defer();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index a3b7bf2..ae90c82 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -332,4 +332,27 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
};
}
};
+}).directive('kylinpopover', function ($compile,$templateCache) {
+ return {
+ restrict: "A",
+ link: function (scope, element, attrs) {
+ var popOverContent;
+ var dOptions = {
+ placement : 'right'
+ }
+ popOverContent = $templateCache.get(attrs.template);
+
+ var placement = attrs.placement? attrs.placement : dOptions.placement;
+ var title = attrs.title;
+
+ var options = {
+ content: popOverContent,
+ placement: placement,
+ trigger: "hover",
+ title: title,
+ html: true
+ };
+ $(element).popover(options);
+ }
+ };
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 3f97254..784b081 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -88,7 +88,8 @@ KylinApp.constant('cubeConfig', {
],
partitionDateFormatOpt:[
'yyyy-MM-dd',
- 'yyyyMMdd'
+ 'yyyyMMdd',
+ 'yyyy-MM-dd HH:mm:ss'
],
partitionTimeFormatOpt:[
'HH:mm:ss',
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/less/app.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 238e72e..8380800 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -792,3 +792,36 @@ input:-moz-placeholder {
.panel-group .panel{
overflow: auto !important;
}
+
+.middle-popover .popover{
+ width: 360px;
+ max-width: 360px;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-size: 14px;
+ line-height: 1.428571429;
+ color:#808080;
+ background-color: #ffffff;
+}
+
+.middle-popover .popover-content, .large-popover .popover-content{
+ font-size: 13px;
+ font-weight: normal;
+ line-height: 18px;
+ display: inline-flex;
+ display: -webkit-inline-flex;
+ word-break: normal;
+}
+
+.middle-popover .popover-content ol, .large-popover .popover-content ol{
+ -webkit-padding-start: 20px;
+ padding-start: 20px;
+}
+.large-popover .popover{
+ width: 600px;
+ max-width: 600px;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-size: 14px;
+ color:#808080;
+ line-height: 1.428571429;
+ background-color: #ffffff;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index d59ad0c..0e1f972 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -21,9 +21,16 @@
<ng-form name="forms.cube_setting_form">
<div>
-
<div class="form-group">
- <h4 style="margin-left:42px">Aggregation Groups</h4>
+ <h3 style="margin-left:42px;">Aggregation Groups </h3>
+ <div style="margin-left:22px;">
+ <div class="box-body">
+ <div class="col-xs-12">
+ Visit <a href="http://kylin.apache.org/blog/2016/02/18/new-aggregation-group" target="_blank">aggregation group</a> for more about aggregation group.
+ </div>
+ </div>
+ </div>
+
<table style="margin-left:42px; width:92%"
ng-if="cubeMetaFrame.aggregation_groups.length > 0"
class="table table-hover list">
@@ -200,85 +207,97 @@
</div>
- <div class="form-group">
- <h4 style="margin-left:42px">Rowkeys</h4>
- <table style="margin-left:42px; width:92%"
- ng-if="cubeMetaFrame.rowkey.rowkey_columns.length > 0"
- class="table table-hover table-bordered list"
- >
- <thead>
- <tr>
- <th>ID</th>
- <th style="width:200px;">Column</th>
- <th>Encoding</th>
- <th>Length</th>
- <th>Shard By</th>
- <th ng-if="state.mode=='edit'"></th>
- </tr>
- </thead>
+ <div class="form-group large-popover">
+ <!--<h4 style="margin-left:42px">-->
+ <!--Rowkeys <i class="fa fa-info-circle" style="white-space:pre-wrap" popover-trigger="mouseenter" popover=" 'dict' encoding will try to build a dictionary for the dimension. 'fixed_length' encoding will encode the dimension vlaues by fixed length bytes with padding. 'int' encoding uses variable integer to encode integers"></i>-->
+ <!--</h4>-->
+ <h3 style="margin-left:42px">Rowkeys <i kylinpopover placement="right" title="Rowkey" template="rowkeyTip.html" class="fa fa-info-circle"></i></h3>
+
+ <div style="margin-left:42px">
+ <div class="box-body">
+ <br/>
+
+
+ <table style="margin-left:width:92%"
+ ng-if="cubeMetaFrame.rowkey.rowkey_columns.length > 0"
+ class="table table-hover table-bordered list">
+ <thead>
+ <tr>
+ <th>ID</th>
+ <th style="width:200px;">Column</th>
+ <th>Encoding</th>
+ <th>Length</th>
+ <th>Shard By</th>
+ <th ng-if="state.mode=='edit'"></th>
+ </tr>
+ </thead>
- <tbody ui-sortable="{update:resortRowkey()}" ng-model="convertedRowkeys">
+ <tbody ui-sortable="{update:resortRowkey()}" ng-model="convertedRowkeys">
- <tr ng-repeat="rowkey_column in convertedRowkeys track by $index"
- ng-class="state.mode=='edit'?'sort-item':''">
+ <tr ng-repeat="rowkey_column in convertedRowkeys track by $index"
+ ng-class="state.mode=='edit'?'sort-item':''">
- <td>
- <!-- ID -->
- <span class="ng-binding" ng-class="state.mode=='edit'?'badge':''">{{($index + 1)}}</span>
- </td>
- <td>
- <!--Column Name -->
- <input type="text" class="form-control" placeholder="Column Name.." ng-if="state.mode=='edit'"
- ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column)"
- tooltip="rowkey column name.." tooltip-trigger="focus"
- ng-model="rowkey_column.column" class="form-control">
-
- <span ng-if="state.mode=='view'">{{rowkey_column.column}}</span>
- </td>
- <td>
- <select ng-if="state.mode=='edit'" style="width:180px;"
- chosen ng-model="rowkey_column.encoding"
- ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);"
- ng-options="dt as dt for dt in store.supportedEncoding">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{rowkey_column.encoding}}</span>
+ <td>
+ <!-- ID -->
+ <span class="ng-binding" ng-class="state.mode=='edit'?'badge':''">{{($index + 1)}}</span>
+ </td>
+ <td>
+ <!--Column Name -->
+ <input type="text" class="form-control" placeholder="Column Name.." ng-if="state.mode=='edit'"
+ ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column)"
+ tooltip="rowkey column name.." tooltip-trigger="focus"
+ ng-model="rowkey_column.column" class="form-control">
+
+ <span ng-if="state.mode=='view'">{{rowkey_column.column}}</span>
+ </td>
+ <td>
+ <select ng-if="state.mode=='edit'" style="width:180px;"
+ chosen ng-model="rowkey_column.encoding"
+ ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);"
+ ng-options="dt as dt for dt in store.supportedEncoding">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{rowkey_column.encoding}}</span>
- </td>
- <td>
- <!--Column Length -->
- <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
- tooltip="rowkey column length.." tooltip-trigger="focus"
- ng-disabled="rowkey_column.encoding=='dict'"
- ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);"
- ng-model="rowkey_column.valueLength" class="form-control">
-
- <small class="help-block red" ng-show="state.mode=='edit' && rowkey_column.encoding=='int' && (rowkey_column.valueLength>8 || rowkey_column.valueLength<1)">int encoding column length should between 1 and 8</small>
- <span ng-if="state.mode=='view'">{{rowkey_column.valueLength}}</span>
- </td>
+ </td>
+ <td>
+ <!--Column Length -->
+ <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
+ tooltip="rowkey column length.." tooltip-trigger="focus"
+ ng-disabled="rowkey_column.encoding=='dict'"
+ ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);"
+ ng-model="rowkey_column.valueLength" class="form-control">
+
+ <small class="help-block red" ng-show="state.mode=='edit' && rowkey_column.encoding=='int' && (rowkey_column.valueLength>8 || rowkey_column.valueLength<1)">int encoding column length should between 1 and 8</small>
+ <span ng-if="state.mode=='view'">{{rowkey_column.valueLength}}</span>
+ </td>
- <td>
- <select ng-if="state.mode=='edit'" style="width:180px;"
- chosen ng-model="rowkey_column.isShardBy"
- tooltip="false by default"
- data-placeholder="false by default"
- ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column,true)"
- ng-options="dt as dt for dt in cubeConfig.rowKeyShardOptions">
- <option value=""></option>
- </select>
- <small class="help-block red" ng-show="state.mode=='edit' && rule.shardColumnAvailable==false && rowkey_column.isShardBy == true">at most one 'shard by' column is allowed.</small>
- <span ng-if="state.mode=='view'">{{rowkey_column.isShardBy}}</span>
- </td>
+ <td>
+ <select ng-if="state.mode=='edit'" style="width:180px;"
+ chosen ng-model="rowkey_column.isShardBy"
+ tooltip="false by default"
+ data-placeholder="false by default"
+ ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column,true)"
+ ng-options="dt as dt for dt in cubeConfig.rowKeyShardOptions">
+ <option value=""></option>
+ </select>
+ <small class="help-block red" ng-show="state.mode=='edit' && rule.shardColumnAvailable==false && rowkey_column.isShardBy == true">at most one 'shard by' column is allowed.</small>
+ <span ng-if="state.mode=='view'">{{rowkey_column.isShardBy}}</span>
+ </td>
+
+ <td ng-if="state.mode=='edit'">
+ <button class="btn btn-xs btn-info"
+ ng-click="removeRowkey(convertedRowkeys, $index,rowkey_column)"><i
+ class="fa fa-minus"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
- <td ng-if="state.mode=='edit'">
- <button class="btn btn-xs btn-info"
- ng-click="removeRowkey(convertedRowkeys, $index,rowkey_column)"><i
- class="fa fa-minus"></i>
- </button>
- </td>
- </tr>
- </tbody>
- </table>
+
+ </div>
+ </div>
<button class="btn btn-sm btn-info" style="margin-left:42px"
ng-click="addNewRowkeyColumn()" ng-show="state.mode=='edit'">New Rowkey Column<i class="fa fa-plus"></i>
@@ -287,13 +306,28 @@
</div>
- <div class="box box-solid" style="margin-left:22px;">
- <div class="box-body">
- <div class="col-xs-12">
- Visit <a href="http://kylin.apache.org/blog/2016/02/18/new-aggregation-group" target="_blank">aggregation group</a> for more about aggregation group.
- </div>
- </div>
- </div>
</ng-form>
</div>
+
+<script type="text/ng-template" id="rowkeyTip.html">\u2028\u2028
+ <div>
+ <h4>Shard By this column?</h4>
+ <p>
+ If specified as "true", cube data will be sharded according to its value.
+ </p>
+
+ <h4>RowKey Encoding</h4>
+ <ol>
+ <li>
+ "dict" encoding will try to build a dictionary for the dimension
+ </li>
+ <li>
+ "fixed_length" encoding will encode the dimension vlaues by fixed length bytes with padding
+ </li>
+ <li>
+ "int" encoding uses variable integer to encode integers
+ </li>
+ </ol>
+ </div>
+</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/cubeDesigner/info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/info.html b/webapp/app/partials/cubeDesigner/info.html
index ec9fc49..d2bc235 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -17,12 +17,12 @@
-->
<div class="row">
- <div class="col-xs-8">
+ <div class="col-xs-12">
<ng-form name="forms.cube_info_form" novalidate>
<!--Project-->
<div class="form-group" ng-class="{'required':state.mode=='edit'}">
<div class="row">
- <label class="col-xs-12 col-sm-3 control-label no-padding-right">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
<b>Model Name</b>
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.model_name.$invalid && (forms.cube_info_form.model_name.$dirty||forms.cube_info_form.model_name.$submitted)}">
@@ -49,10 +49,10 @@
</div>
<!--Cube Name-->
- <div class="form-group" ng-class="{'required':state.mode=='edit'}">
+ <div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Cube Name</b>
+ <b>Cube Name</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Cube Name" template="cubeNameTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.cube_info_form.cube_name.$invalid && (forms.cube_info_form.cube_name.$dirty||forms.cube_info_form.cube_name.$submitted)}">
<input ng-if="state.mode=='edit'" ng-disabled="{{isEdit}}" name="cube_name" type="text" class="form-control"
@@ -71,7 +71,7 @@
<div class="form-group">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Notification List</b>
+ <b>Notification Email List</b>
</label>
<div class="col-xs-12 col-sm-6">
<input type="text" class="form-control" placeholder="Comma Separated"
@@ -82,15 +82,16 @@
</div>
<div class="form-group">
- <div class="row">
+ <div class="row middle-popover">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Notification Status</b>
+ <b>Notification Events</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Notification Events" template="notificationEventsTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6">
<ui-select
autofocus="true"
close-on-select="false"
+ on-remove="removeNotificationEvents()"
ng-if="state.mode=='edit'" style="width: 100%" multiple ng-model="cubeMetaFrame.status_need_notify">
<ui-select-match placeholder="Select Status...">{{$item}}</ui-select-match>
<ui-select-choices
@@ -120,24 +121,21 @@
</ng-form>
</div>
- <!--Tips-->
- <div class="col-xs-4">
- <div class="box box-solid">
- <div class="box-header widget-header-flat">
- <h4 class="box-title">Tips</h4>
- </div>
- <div class="box-body">
- <div class="row">
- <div class="col-xs-12">
- <ol class="text-info">
- <li>Cube name is unique name of entire system</li>
- <li>Can not edit cube name after created</li>
- <li>Cube must belong to project which you have privilege to create</li>
- <li>Error status will notify by default</li>
- </ol>
- </div>
- </div>
- </div>
- </div>
- </div>
</div>
+
+<script id="tip-cube-name.html" type="text/ng-template">
+ <strong>HTML</strong> allowed.<br />
+ Binding allowed: {{ value }}<br />
+ <button ng-click="value = 2">Test</button>
+</script>
+
+<script type="text/ng-template" id="cubeNameTip.html">\u2028\u2028
+<ol>
+ <li>Cube name is unique name of entire system.</li>
+ <li>Can not edit cube name after created.</li>
+</ol>
+</script>
+
+<script type="text/ng-template" id="notificationEventsTip.html">\u2028\u2028
+ <p>ERROR status will trigger a notification anyway.</p>
+</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index f0ae86f..9f12317 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -97,9 +97,9 @@
</div>
</div>
<!--Expression-->
- <div class="form-group">
+ <div class="form-group middle-popover">
<div class="row">
- <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Expression</b></label>
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Expression</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Expression" template="expressionTip.html"></i></label>
<div class="col-xs-12 col-sm-6">
<select class="form-control"
ng-init="newMeasure.function.expression = (!!newMeasure.function.expression)?newMeasure.function.expression:cubeConfig.dftSelections.measureExpression" chosen ng-model="newMeasure.function.expression" required
@@ -130,12 +130,12 @@
</div>
</div>
<!--Param Value-->
- <div class="form-group">
+ <div class="form-group middle-popover">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Param Value</b>
+ <b>Param Value</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Param Value" template="paramvalueTip.html"></i>
<!--tip for top_n-->
- <small ng-if="newMeasure.function.expression == 'TOP_N'" class="help-block" style="color:#b94a48">(SUM|ORDER BY Column for TOP_N)</small>
+ <small ng-if="newMeasure.function.expression == 'TOP_N'" class="help-block" style="color:#3a87ad">(SUM|ORDER BY Column for TOP_N)</small>
</label>
@@ -232,25 +232,6 @@
</div>
</div>
- <!--Tips-->
- <div class="col-xs-4">
- <div class="box box-solid">
- <div class="box-header">
- <h4 class="box-title">Tips</h4>
- </div>
- <div class="box-body">
- <div class="row">
- <div class="col-xs-12">
- <ol class="text-info">
- <li>All cubes have to contain one measure for Count(1), suggest use "_Count_" as name (Has been generated automatically)</li>
- <li>Only accept single column in param value with "Column" type</li>
- <li>Distinct Count is approximate, please indicate Error Rate, higher accuracy degree accompanied with larger storage size and longer build time</li>
- </ol>
- </div>
- </div>
- </div>
- </div>
- </div>
</div>
</div>
<div class="box-footer">
@@ -339,3 +320,14 @@
<button class="btn btn-primary" ng-click="cancel()">Close</button>
</div>
</script>
+
+<script type="text/ng-template" id="expressionTip.html">\u2028\u2028
+ <p>All cubes have to contain one measure for Count(1), suggest use "_Count_" as name (Has been generated automatically)</p>
+</script>
+
+<script type="text/ng-template" id="paramvalueTip.html">\u2028\u2028
+ <ol>
+ <li>Only accept single column in param value with "Column" type</li>
+ <li>Distinct Count is approximate, please indicate Error Rate, higher accuracy degree accompanied with larger storage size and longer build time</li>
+ </ol>
+</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/cubeDesigner/refresh_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/refresh_settings.html b/webapp/app/partials/cubeDesigner/refresh_settings.html
index 814c453..4c18639 100755
--- a/webapp/app/partials/cubeDesigner/refresh_settings.html
+++ b/webapp/app/partials/cubeDesigner/refresh_settings.html
@@ -28,9 +28,11 @@
<td>
<!--Cube Size-->
<div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Auto Merge Time
- Ranges(days)</b></label>
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Auto Merge Thresholds</b>
+ <i class="fa fa-info-circle" kylinpopover title="Auto Merge Thresholds" template="autoMergeThresholdsTip.html" placement="right"></i>
+ </label>
<div class="col-xs-12 col-sm-6">
<!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
@@ -71,7 +73,7 @@
</table>
<button class="btn btn-xs btn-info" ng-show="state.mode=='edit'"
ng-click="addNewMergeTimeRange()">
- New Merge Range<i class="fa fa-plus"></i>
+ New Thresholds <i class="fa fa-plus"></i>
</button>
</div>
<div class="row" ng-if="state.mode=='view'">
@@ -96,18 +98,16 @@
<td>
<!--Cube Size-->
<div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Retention
- Range(days)</b></label>
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Retention Threshold</b> <i class="fa fa-info-circle" title="Retention Threshold" kylinpopover template="retentionTip.html" placement="right"></i>
+ </label>
<div class="col-xs-12 col-sm-6">
<!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
<input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid"
placeholder="how many days cube retention" ng-model="cubeMetaFrame.retention_range"
ng-if="state.mode=='edit'">
- <small class="help-block text-red" ng-show="state.mode=='edit'">(by default it's '0',which will keep
- all historic cube segments ,or will keep latest [Retention Range] days cube segments )
- </small>
<span ng-if="state.mode=='view'">{{cubeMetaFrame.retention_range | millisecondsToDay}}</span>
</div>
</div>
@@ -154,3 +154,11 @@
</div>
</ng-form>
</div>
+
+<script type="text/ng-template" id="autoMergeThresholdsTip.html">\u2028\u2028
+ <p>The thresholds will be checked ascendingly to see if any consectuive segments' time range has exceeded it. For example the [7 days, 30 days] will result in daily incremental segments being merged every 7 days, and the 7-days segments will get merged every 30 days.</p>
+</script>
+
+<script type="text/ng-template" id="retentionTip.html">\u2028\u2028
+<p>By default it's '0',which will keep all historic cube segments ,or will keep latest [Retention Threshold] days cube segments.</p>
+</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 2a46698..d000aab 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -17,121 +17,119 @@
-->
<div class="row">
- <div class="col-xs-8">
+ <div class="col-xs-12">
<ng-form name="forms.model_setting_form" novalidate>
+
+ <h3>Partition</h3>
+ <div class="box-body">
<!--Cube Partition Type-->
- <div class="form-group" ng-hide="true">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Type</b></label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control"
- ng-if="state.mode=='edit'"
- chosen ng-model="modelsManager.selectedModel.partition_desc.partition_type"
- ng-options="ddt as ddt for ddt in cubeConfig.cubePartitionTypes">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.partition_desc.partition_type}}</span>
- </div>
- </div>
- </div>
+ <div class="form-group" ng-hide="true">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Type</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select class="form-control"
+ ng-if="state.mode=='edit'"
+ chosen ng-model="modelsManager.selectedModel.partition_desc.partition_type"
+ ng-options="ddt as ddt for ddt in cubeConfig.cubePartitionTypes">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.partition_desc.partition_type}}</span>
+ </div>
+ </div>
+ </div>
- <!--Partition Column-->
- <div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b></label>
- <div class="col-xs-12 col-sm-6">
+ <!--Partition Column-->
+ <div class="form-group">
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b> <i kylinpopover placement="right" title="Partition Date Column" template="partitionTip.html" class="fa fa-info-circle"></i></label>
+ <div class="col-xs-12 col-sm-6">
- <select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
- ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
- ng-if="state.mode=='edit'"
- data-placement=""
- ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonColumns(modelsManager.selectedModel.fact_table)" >
- <option value="">--Select Partition Column--</option>
- </select>
- <small class="help-block text-red" ng-show="state.mode=='edit'">(Column Type should be DATE or TIMESTAMP Type)</small>
-
- <span ng-if="state.mode=='view'">
- {{!!(modelsManager.selectedModel.partition_desc.partition_date_column)?modelsManager.selectedModel.partition_desc.partition_date_column: ''}}</span>
- </div>
- </div>
- </div>
+ <select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
+ ng-if="state.mode=='edit'"
+ data-placement=""
+ ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonColumns(modelsManager.selectedModel.fact_table)" >
+ <option value="">--Select Partition Column--</option>
+ </select>
+ <!--<small class="text-info" ng-show="state.mode=='edit'">(Column Type should be DATE or TIMESTAMP Type)</small>-->
- <!--Date Format-->
- <div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
- <div class="col-xs-12 col-sm-6">
- <select style="width: 100%" chosen
- ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-if="state.mode=='edit'"
- data-placement=""
- ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
- <option value="">--Select Date Format--</option>
- </select>
- <span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_date_column">{{(modelsManager.selectedModel.partition_desc.partition_date_format)}}</span>
- </div>
+ <span ng-if="state.mode=='view'">
+ {{!!(modelsManager.selectedModel.partition_desc.partition_date_column)?modelsManager.selectedModel.partition_desc.partition_date_column: ''}}</span>
+ </div>
+ </div>
</div>
- </div>
- <!--Partition Time Column-->
- <div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Time Column</b></label>
- <div class="col-xs-12 col-sm-6">
-
- <select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
- ng-model="modelsManager.selectedModel.partition_desc.partition_time_column"
- ng-if="state.mode=='edit'"
- data-placement=""
- ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonTimeColumns(modelsManager.selectedModel.fact_table)" >
- <option value="">--Select Partition Column--</option>
- </select>
- <small class="help-block text-red" ng-show="state.mode=='edit'">(Column Type should be TIME Type)</small>
-
- <span ng-if="state.mode=='view'">
- {{!!(modelsManager.selectedModel.partition_desc.partition_time_column)?modelsManager.selectedModel.partition_desc.partition_time_column: ''}}</span>
+ <!--Date Format-->
+ <div class="form-group">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select style="width: 100%" chosen
+ ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
+ ng-if="state.mode=='edit'"
+ data-placement=""
+ ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
+ <option value="">--Select Date Format--</option>
+ </select>
+ <span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_date_column">{{(modelsManager.selectedModel.partition_desc.partition_date_format)}}</span>
+ </div>
</div>
</div>
- </div>
- <!--Time Format-->
- <div class="form-group">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Time Format</b></label>
- <div class="col-xs-12 col-sm-6">
- <select style="width: 100%" chosen
- ng-required="modelsManager.selectedModel.partition_desc.partition_time_format"
- ng-model="modelsManager.selectedModel.partition_desc.partition_time_format"
- ng-if="state.mode=='edit'"
- data-placement=""
- ng-options="ddt as ddt for ddt in cubeConfig.partitionTimeFormatOpt">
- <option value="">--Select Time Format--</option>
- </select>
- <span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_time_column">{{(modelsManager.selectedModel.partition_desc.partition_time_format)}}</span>
+ <!--Date Format-->
+ <div class="form-group middle-popover">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has a separate "time of the day" column ?</b> <i kylinpopover placement="right" title="Separate Time Column" template="separateTimeColumnTip.html" class="fa fa-info-circle"></i></label>
+ <div class="col-xs-12 col-sm-6">
+ <toggle-switch on-label="Yes" off-label="No" ng-disabled="state.mode!=='edit'" ng-click="toggleHasSeparateColumn()" ng-model="partitionColumn.hasSeparateTimeColumn"></toggle-switch>
+ </div>
</div>
</div>
- </div>
+ <!--Partition Time Column-->
+ <div class="form-group" ng-if="partitionColumn.hasSeparateTimeColumn == true">
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Time Column</b> <i kylinpopover placement="right" title="Partition Time Column" template="partitionTimeTip.html" class="fa fa-info-circle"></i></label>
+ <div class="col-xs-12 col-sm-6">
- <div class="form-group" ng-show="userService.hasRole('ROLE_ADMIN')">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Cube Size</b></label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control"
- ng-if="state.mode=='edit'"
- chosen ng-model="modelsManager.selectedModel.capacity"
- ng-options="ddt as ddt for ddt in cubeConfig.cubeCapacities">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.capacity}}</span>
- </div>
+ <select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_time_column"
+ ng-if="state.mode=='edit'"
+ data-placement=""
+ ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonTimeColumns(modelsManager.selectedModel.fact_table)" >
+ <option value="">--Select Partition Column--</option>
+ </select>
+ <!--<small class="text-info" ng-show="state.mode=='edit'">(Column Type should be TIME Type)</small>-->
+
+ <span ng-if="state.mode=='view'">
+ {{!!(modelsManager.selectedModel.partition_desc.partition_time_column)?modelsManager.selectedModel.partition_desc.partition_time_column: ''}}</span>
</div>
+ </div>
</div>
+ <!--Time Format-->
+ <div class="form-group" ng-if="partitionColumn.hasSeparateTimeColumn == true">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Time Format</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select style="width: 100%" chosen
+ ng-required="modelsManager.selectedModel.partition_desc.partition_time_format"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_time_format"
+ ng-if="state.mode=='edit'"
+ data-placement=""
+ ng-options="ddt as ddt for ddt in cubeConfig.partitionTimeFormatOpt">
+ <option value="">--Select Time Format--</option>
+ </select>
+ <span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_time_column">{{(modelsManager.selectedModel.partition_desc.partition_time_format)}}</span>
+ </div>
+ </div>
+ </div>
+ </div>
+ <h3>Filter</h3>
<!--Filter Condition-->
- <div class="form-group">
+ <div class="form-group box-body middle-popover">
<div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Filter</b></label>
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Filter</b> <i kylinpopover placement="right" title="Filter" template="filterTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6">
<!--edit mode-->
@@ -161,26 +159,27 @@
</div>
</ng-form>
</div>
-
- <div class="col-xs-4">
- <div class="box box-solid">
- <div class="box-header">
- <h4 class="box-title">Tips</h4>
- </div>
- <div class="box-body">
- <div class="row">
- <div class="col-xs-12">
- <ol class="text-info">
- <li>Partition date column not required,leave as default if cube always need full build</li>
- <li>Partition date column will select 'date' type column from fact table</li>
- <li>Where clause to filter data from source</li>
- <li>Do not include date column which will be used for incremental refresh</li>
- <li>Do not include "Where"</li>
- <li>Please verify SQL when finish cube design from SQL view of cube</li>
- </ol>
- </div>
- </div>
- </div>
- </div>
- </div>
</div>
+
+<script type="text/ng-template" id="partitionTip.html">
+ <ol>
+ <li>Partition date column not required,leave as default if cube always need full build</Li>
+ <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR, etc.)</li>
+ </ol>
+</script>
+
+<script type="text/ng-template" id="partitionTimeTip.html">\u2028\u2028
+ <p>Column should contain time value (type can be Timestamp, String, VARCHAR, etc.)</p>
+</script>
+
+<script typ type="text/ng-template" id="filterTip.html">\u2028\u2028
+ <ol>
+ <li>Where clause to filter data from source</li>
+ <li>Do not include date column which will be used for incremental refresh</li>
+ <li>Do not include "Where"</li>
+ </ol>
+</script>
+
+<script type="text/ng-template" id="separateTimeColumnTip.html">\u2028\u2028
+<p>For cases where fact table saves date and hour at two columns (KYLIN-1427)</p>
+</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f0ed10e/webapp/app/partials/modelDesigner/model_info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_info.html b/webapp/app/partials/modelDesigner/model_info.html
index a6ad300..cab710b 100644
--- a/webapp/app/partials/modelDesigner/model_info.html
+++ b/webapp/app/partials/modelDesigner/model_info.html
@@ -17,15 +17,15 @@
-->
<div class="row">
- <div class="col-xs-8">
+ <div class="col-xs-12">
<!--Project-->
<ng-form name="forms.model_info_form" novalidate>
<!--Model Name-->
- <div class="form-group" ng-class="{'required':state.mode=='edit'}">
+ <div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
- <b>Model Name</b>
+ <b>Model Name</b> <i class="fa fa-info-circle" kylinpopover placement="right" title="Model Name" template="modelNameTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.model_info_form.model_name.$invalid && (forms.model_info_form.model_name.$dirty||forms.model_info_form.$submitted)}">
@@ -61,24 +61,6 @@
</ng-form>
</div>
- <!--Tips-->
- <div class="col-xs-4">
- <div class="box box-solid">
- <div class="box-header widget-header-flat">
- <h4 class="box-title">Tips</h4>
- </div>
- <div class="box-body">
- <div class="row">
- <div class="col-xs-12">
- <ol class="text-info">
- <li>Model must belong to project which you have privilege to create</li>
- <li>Model name is unique name of entire system</li>
- </ol>
- </div>
- </div>
- </div>
- </div>
- </div>
</div>
<div class="row" ng-hide="true">
@@ -97,3 +79,10 @@
</div>
</div>
</div>
+
+<script type="text/ng-template" id="modelNameTip.html">\u2028\u2028
+<ol>
+ <li>Model name is unique name of entire system.</li>
+ <li>Can not edit model name after created.</li>
+</ol>
+</script>
[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 12b1fa8..3a67d0c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -35,8 +35,9 @@ import javax.annotation.Nullable;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -267,7 +268,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -339,7 +340,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
Map<byte[], CubeVisitProtos.CubeVisitResponse> results;
try {
- results = getResults(builder.build(), conn.getTable(cubeSeg.getStorageLocationIdentifier()), epRange.getFirst(), epRange.getSecond());
+ results = getResults(builder.build(), conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier())), epRange.getFirst(), epRange.getSecond());
} catch (Throwable throwable) {
throw new RuntimeException(logHeader + "Error when visiting cubes by endpoint", throwable);
}
@@ -384,7 +385,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
- private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, HTableInterface table, byte[] startKey, byte[] endKey) throws Throwable {
+ private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, Table table, byte[] startKey, byte[] endKey) throws Throwable {
Map<byte[], CubeVisitProtos.CubeVisitResponse> results = table.coprocessorService(CubeVisitProtos.CubeVisitService.class, startKey, endKey, new Batch.Call<CubeVisitProtos.CubeVisitService, CubeVisitProtos.CubeVisitResponse>() {
public CubeVisitProtos.CubeVisitResponse call(CubeVisitProtos.CubeVisitService rowsService) throws IOException {
ServerRpcController controller = new ServerRpcController();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 938145b..9b8a3de 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,8 +24,9 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -164,8 +165,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 0cd35f1..c96d706 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -138,7 +138,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -174,7 +174,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
debugGitTag = region.getTableDesc().getValue(IRealizationConstants.HTableGitTag);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
index 9c96f21..a277eeb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
@@ -24,7 +24,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
@@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.IIDeployCoprocessorCLI;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
@@ -47,7 +49,7 @@ public class IICreateHTableJob extends AbstractHadoopJob {
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
options.addOption(OPTION_II_NAME);
options.addOption(OPTION_HTABLE_NAME);
@@ -63,10 +65,11 @@ public class IICreateHTableJob extends AbstractHadoopJob {
Configuration conf = HBaseConfiguration.create(getConf());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// check if the table already exists
- admin = new HBaseAdmin(conf);
- if (admin.tableExists(tableName)) {
- if (admin.isTableEnabled(tableName)) {
+ admin = conn.getAdmin();
+ if (admin.tableExists(TableName.valueOf(tableName))) {
+ if (admin.isTableEnabled(TableName.valueOf(tableName))) {
logger.info("Table " + tableName + " already exists and is enabled, no need to create.");
return 0;
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
index fef9662..fa782bf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
@@ -20,7 +20,7 @@ package org.apache.kylin.storage.hbase.ii;
import java.util.ArrayList;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -53,8 +53,8 @@ public class InvertedIndexStorageQuery implements IStorageQuery {
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
String tableName = seg.getStorageLocationIdentifier();
- //HConnection is cached, so need not be closed
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ //Connection is cached, so need not be closed
+ Connection conn = HBaseConnection.get(context.getConnUrl());
try {
dataIterator = new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn, returnTupleInfo);
return dataIterator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 4ec421b..8da8e59 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -31,8 +31,9 @@ import javax.annotation.Nullable;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SerializationUtils;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -93,16 +94,16 @@ public class EndpointTupleIterator implements ITupleIterator {
private Iterator<List<IIProtos.IIResponseInternal.IIRow>> regionResponsesIterator = null;
private ITupleIterator tupleIterator = null;
- private HTableInterface table = null;
+ private Table table = null;
private TblColRef partitionCol;
private long lastDataTime = -1;
private int rowsInAllMetric = 0;
- public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn, TupleInfo returnTupleInfo) throws Throwable {
+ public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, Connection conn, TupleInfo returnTupleInfo) throws Throwable {
String tableName = segment.getStorageLocationIdentifier();
- table = conn.getTable(tableName);
+ table = conn.getTable(TableName.valueOf(tableName));
factTableName = segment.getIIDesc().getFactTableName();
if (rootFilter == null) {
@@ -290,7 +291,7 @@ public class EndpointTupleIterator implements ITupleIterator {
return request;
}
- private Collection<IIProtos.IIResponse> getResults(final IIProtos.IIRequest request, HTableInterface table) throws Throwable {
+ private Collection<IIProtos.IIResponse> getResults(final IIProtos.IIRequest request, Table table) throws Throwable {
Map<byte[], IIProtos.IIResponse> results = table.coprocessorService(IIProtos.RowsService.class, null, null, new Batch.Call<IIProtos.RowsService, IIProtos.IIResponse>() {
public IIProtos.IIResponse call(IIProtos.RowsService rowsService) throws IOException {
ServerRpcController controller = new ServerRpcController();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index 48802cb..0b5a963 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -91,7 +91,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
if (request.hasTsRange()) {
Range<Long> tsRange = (Range<Long>) SerializationUtils.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getTsRange()));
- byte[] regionStartKey = region.getStartKey();
+ byte[] regionStartKey = region.getRegionInfo().getStartKey();
if (!ArrayUtils.isEmpty(regionStartKey)) {
shard = BytesUtil.readUnsigned(regionStartKey, 0, IIKeyValueCodec.SHARD_LEN);
} else {
@@ -145,7 +145,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
HRegion region = null;
try {
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
innerScanner = region.getScanner(prepareScan(request, region));
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 14f47ef..455d203 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -25,7 +25,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
admin.close();
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 472b638..9dc9715 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -27,11 +27,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -100,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index ddc868d..5063e00 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -36,7 +36,7 @@ package org.apache.kylin.storage.hbase.steps;
import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -63,7 +63,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -72,7 +72,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -131,7 +131,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index fa62a62..cf5c9d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -28,7 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -55,7 +56,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
try {
CubeSegment cubeSegment = (CubeSegment) buildable;
- final HTableInterface hTable;
+ final Table hTable;
hTable = createHTable(cubeSegment);
List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -87,10 +88,10 @@ public class HBaseStreamingOutput implements IStreamingOutput {
}
}
- private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException {
+ private Table createHTable(final CubeSegment cubeSegment) throws IOException {
final String hTableName = cubeSegment.getStorageLocationIdentifier();
CubeHTableUtil.createHTable(cubeSegment, null);
- final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+ final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!");
return hTable;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index a1377f2..dfde2dd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,12 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -70,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 1ba96e9..8c19b9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,11 +21,12 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -53,8 +54,8 @@ public class CleanHtableCLI extends AbstractHadoopJob {
}
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -65,7 +66,7 @@ public class CleanHtableCLI extends AbstractHadoopJob {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 5acecbc..05eef7a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -73,7 +73,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -117,8 +117,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -142,6 +142,10 @@ public class CubeMigrationCLI {
} else {
showOpts();
}
+
+ checkMigrationSuccess(dstConfig, cubeName, true);
+
+ IOUtils.closeQuietly(hbaseAdmin);
}
public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
@@ -311,10 +315,10 @@ public class CubeMigrationCLI {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
logger.info("CHANGE_HTABLE_HOST is completed");
break;
}
@@ -425,11 +429,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -445,11 +449,10 @@ public class CubeMigrationCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
+ put.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), CellUtil.cloneValue(cell));
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -476,10 +479,10 @@ public class CubeMigrationCLI {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
break;
}
case COPY_FILE_IN_META: {
@@ -509,13 +512,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index f1fc8e1..b735864 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -27,7 +27,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +63,7 @@ public class CubeMigrationCheckCLI {
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -129,9 +131,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -188,10 +189,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable "+sepNameList[0]+"belonging to cube "+sepNameList[1]+" from "+desc.getValue(IRealizationConstants.HTableTag)+" to "+dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
}else{
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 270da13..f742e3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -80,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -163,10 +166,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,9 +192,9 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static void resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
logger.info("Unset coprocessor on " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -219,13 +222,13 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
}
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(Admin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processed = new ArrayList<String>();
for (String tableName : tableNames) {
@@ -336,7 +339,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index c55bde4..ad5b647 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -26,8 +26,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
@@ -232,9 +233,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,7 +255,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 8a88b6d..760ed3a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,9 +28,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 4ce0dac..f5b315d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -29,8 +29,12 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -79,8 +83,8 @@ public class HBaseClean extends AbstractHadoopJob {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -95,12 +99,12 @@ public class HBaseClean extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 346c3a2..58aa8fd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -20,22 +20,11 @@
package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -57,30 +46,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -104,7 +94,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- hBaseAdmin.close();
+ admin.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 0a791b7..8fb2f58 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -26,12 +26,16 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.kylin.storage.hbase.HBaseConnection;
public class HBaseUsage {
@@ -43,8 +47,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index deb8da1..79f9fb3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,16 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +59,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -119,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -135,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -170,8 +171,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -206,7 +207,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -224,8 +225,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index 2b4a9a7..095743f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -27,8 +27,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -71,8 +74,8 @@ public class HtableAlterMetadataCLI extends AbstractHadoopJob {
}
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index f0618c9..b7303a5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -31,10 +31,15 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,7 +91,8 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -105,12 +111,12 @@ public class OrphanHBaseCleanJob extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 58ef7cb..b86b561 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -21,9 +21,10 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -57,12 +58,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index b1a5ba4..52f71cf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -23,9 +23,10 @@ import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.kylin.common.util.Bytes;
@@ -71,8 +72,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index c010d51..fd39ee6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -35,7 +35,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -52,6 +54,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,7 +107,8 @@ public class StorageCleanupJob extends AbstractHadoopJob {
IIManager iiManager = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -148,12 +152,12 @@ public class StorageCleanupJob extends AbstractHadoopJob {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 23357f5..3fec98c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -26,9 +26,12 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -39,7 +42,6 @@ import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import com.google.common.collect.Lists;
@@ -52,14 +54,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -181,9 +184,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index cd4e33d..9585575 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -29,12 +29,10 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -233,15 +231,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -310,6 +301,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -326,16 +322,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
[5/8] kylin git commit: KYLIN-1722 add serialVersionUID in
IntegerDimEnc
Posted by li...@apache.org.
KYLIN-1722 add serialVersionUID in IntegerDimEnc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a372fe33
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a372fe33
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a372fe33
Branch: refs/heads/1.5.x-HBase1.x
Commit: a372fe336a726d9155d13558b6d3deb23e6e8479
Parents: 8c10e9e
Author: Yang Li <li...@apache.org>
Authored: Sun May 22 13:15:09 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun May 22 13:27:00 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java | 2 ++
.../src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java | 1 +
.../src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java | 1 +
.../src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java | 1 +
4 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a372fe33/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
index 0350eb1..84581ab 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
@@ -26,6 +26,8 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
public class TrimmedDimEnc extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
+
int fixedLen;
//no-arg constructor is required for Externalizable
http://git-wip-us.apache.org/repos/asf/kylin/blob/a372fe33/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 7f80a7f..6dfeaa1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DictionaryDimEnc extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a372fe33/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index 1e9ca79..7805fcc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FixedLenDimEnc extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a372fe33/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 1d821c8..9701447 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -31,6 +31,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class IntegerDimEnc extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class);
[2/8] kylin git commit: KYLIN-1708 tip enhancement,
remove illegal character
Posted by li...@apache.org.
KYLIN-1708 tip enhancement, remove illegal character
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/60a7c747
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/60a7c747
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/60a7c747
Branch: refs/heads/1.5.x-HBase1.x
Commit: 60a7c747ad9ec8ad8b9b25b309e0a6639919efa6
Parents: 5f0ed10
Author: Jason <ji...@163.com>
Authored: Thu May 19 15:07:55 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu May 19 15:07:55 2016 +0800
----------------------------------------------------------------------
webapp/app/index.html | 3 +--
webapp/app/partials/cubeDesigner/advanced_settings.html | 6 +-----
webapp/app/partials/cubeDesigner/info.html | 12 ++----------
webapp/app/partials/cubeDesigner/measures.html | 6 ++----
webapp/app/partials/cubeDesigner/refresh_settings.html | 4 ++--
.../app/partials/modelDesigner/conditions_settings.html | 9 ++++-----
webapp/app/partials/modelDesigner/model_info.html | 2 +-
7 files changed, 13 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/webapp/app/index.html b/webapp/app/index.html
index e99b81a..36245f1 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -260,7 +260,6 @@
<div class="callout callout-info">
<p>{{text}}</p>
</div>
-
-
+</script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 0e1f972..5f18b51 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -208,9 +208,6 @@
<div class="form-group large-popover">
- <!--<h4 style="margin-left:42px">-->
- <!--Rowkeys <i class="fa fa-info-circle" style="white-space:pre-wrap" popover-trigger="mouseenter" popover=" 'dict' encoding will try to build a dictionary for the dimension. 'fixed_length' encoding will encode the dimension vlaues by fixed length bytes with padding. 'int' encoding uses variable integer to encode integers"></i>-->
- <!--</h4>-->
<h3 style="margin-left:42px">Rowkeys <i kylinpopover placement="right" title="Rowkey" template="rowkeyTip.html" class="fa fa-info-circle"></i></h3>
<div style="margin-left:42px">
@@ -309,8 +306,7 @@
</ng-form>
</div>
-
-<script type="text/ng-template" id="rowkeyTip.html">\u2028\u2028
+<script type="text/ng-template" id="rowkeyTip.html">
<div>
<h4>Shard By this column?</h4>
<p>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/cubeDesigner/info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/info.html b/webapp/app/partials/cubeDesigner/info.html
index d2bc235..755fc12 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -122,20 +122,12 @@
</div>
</div>
-
-<script id="tip-cube-name.html" type="text/ng-template">
- <strong>HTML</strong> allowed.<br />
- Binding allowed: {{ value }}<br />
- <button ng-click="value = 2">Test</button>
-</script>
-
-<script type="text/ng-template" id="cubeNameTip.html">\u2028\u2028
+<script type="text/ng-template" id="cubeNameTip.html">
<ol>
<li>Cube name is unique name of entire system.</li>
<li>Can not edit cube name after created.</li>
</ol>
</script>
-
-<script type="text/ng-template" id="notificationEventsTip.html">\u2028\u2028
+<script type="text/ng-template" id="notificationEventsTip.html">
<p>ERROR status will trigger a notification anyway.</p>
</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 9f12317..5778003 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -320,12 +320,10 @@
<button class="btn btn-primary" ng-click="cancel()">Close</button>
</div>
</script>
-
-<script type="text/ng-template" id="expressionTip.html">\u2028\u2028
+<script type="text/ng-template" id="expressionTip.html">
<p>All cubes have to contain one measure for Count(1), suggest use "_Count_" as name (Has been generated automatically)</p>
</script>
-
-<script type="text/ng-template" id="paramvalueTip.html">\u2028\u2028
+<script type="text/ng-template" id="paramvalueTip.html">
<ol>
<li>Only accept single column in param value with "Column" type</li>
<li>Distinct Count is approximate, please indicate Error Rate, higher accuracy degree accompanied with larger storage size and longer build time</li>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/cubeDesigner/refresh_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/refresh_settings.html b/webapp/app/partials/cubeDesigner/refresh_settings.html
index 4c18639..0d88e7c 100755
--- a/webapp/app/partials/cubeDesigner/refresh_settings.html
+++ b/webapp/app/partials/cubeDesigner/refresh_settings.html
@@ -155,10 +155,10 @@
</ng-form>
</div>
-<script type="text/ng-template" id="autoMergeThresholdsTip.html">\u2028\u2028
+<script type="text/ng-template" id="autoMergeThresholdsTip.html">
<p>The thresholds will be checked ascendingly to see if any consectuive segments' time range has exceeded it. For example the [7 days, 30 days] will result in daily incremental segments being merged every 7 days, and the 7-days segments will get merged every 30 days.</p>
</script>
-<script type="text/ng-template" id="retentionTip.html">\u2028\u2028
+<script type="text/ng-template" id="retentionTip.html">
<p>By default it's '0',which will keep all historic cube segments ,or will keep latest [Retention Threshold] days cube segments.</p>
</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index d000aab..a4e49a9 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -168,18 +168,17 @@
</ol>
</script>
-<script type="text/ng-template" id="partitionTimeTip.html">\u2028\u2028
+<script type="text/ng-template" id="partitionTimeTip.html">
<p>Column should contain time value (type can be Timestamp, String, VARCHAR, etc.)</p>
</script>
-<script typ type="text/ng-template" id="filterTip.html">\u2028\u2028
+<script type="text/ng-template" id="filterTip.html">
<ol>
<li>Where clause to filter data from source</li>
<li>Do not include date column which will be used for incremental refresh</li>
<li>Do not include "Where"</li>
</ol>
</script>
-
-<script type="text/ng-template" id="separateTimeColumnTip.html">\u2028\u2028
-<p>For cases where fact table saves date and hour at two columns (KYLIN-1427)</p>
+<script type="text/ng-template" id="separateTimeColumnTip.html">
+ <p>For cases where fact table saves date and hour at two columns (KYLIN-1427)</p>
</script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/60a7c747/webapp/app/partials/modelDesigner/model_info.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_info.html b/webapp/app/partials/modelDesigner/model_info.html
index cab710b..1e83426 100644
--- a/webapp/app/partials/modelDesigner/model_info.html
+++ b/webapp/app/partials/modelDesigner/model_info.html
@@ -80,7 +80,7 @@
</div>
</div>
-<script type="text/ng-template" id="modelNameTip.html">\u2028\u2028
+<script type="text/ng-template" id="modelNameTip.html">
<ol>
<li>Model name is unique name of entire system.</li>
<li>Can not edit model name after created.</li>
[8/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb740ec8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb740ec8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb740ec8
Branch: refs/heads/1.5.x-HBase1.x
Commit: fb740ec889257329028eec2e4c94d69ff32e3258
Parents: 08807fa
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue May 24 14:09:28 2016 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/invertedindex/IITest.java | 10 ++-
.../apache/kylin/common/KylinConfigBase.java | 5 +-
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../test_case_data/sandbox/kylin_job_conf.xml | 86 +++++++++---------
examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
.../kylin/provision/BuildCubeWithEngine.java | 54 +++++------
.../kylin/provision/BuildIIWithStream.java | 13 +--
.../hbase/ii/ITInvertedIndexHBaseTest.java | 9 +-
pom.xml | 18 ++--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 38 +++-----
.../apache/kylin/rest/service/QueryService.java | 21 +++--
.../apache/kylin/rest/service/UserService.java | 16 ++--
.../kylin/storage/hbase/HBaseConnection.java | 36 ++++----
.../kylin/storage/hbase/HBaseResourceStore.java | 29 +++---
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +-
.../hbase/cube/v1/HBaseClientKVIterator.java | 11 +--
.../hbase/cube/v1/RegionScannerAdapter.java | 11 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 15 +++-
.../observer/ObserverAggregationCache.java | 15 ++--
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 11 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/ii/IICreateHTableJob.java | 13 +--
.../hbase/ii/InvertedIndexStorageQuery.java | 6 +-
.../endpoint/EndpointTupleIterator.java | 13 +--
.../ii/coprocessor/endpoint/IIEndpoint.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../hbase/steps/HBaseStreamingOutput.java | 9 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 24 ++---
.../storage/hbase/util/CleanHtableCLI.java | 13 +--
.../storage/hbase/util/CubeMigrationCLI.java | 38 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 23 ++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 42 ++++-----
.../kylin/storage/hbase/util/HBaseUsage.java | 10 ++-
.../storage/hbase/util/HbaseStreamingInput.java | 31 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 18 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 16 ++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 31 +++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
59 files changed, 530 insertions(+), 583 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..5d2cfc4 100644
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.FIFOIterable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
@@ -222,6 +223,11 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
+ public int getBatch() {
+ return -1;
+ }
+
+ @Override
public boolean nextRaw(List<Cell> result) throws IOException {
if (iiRowIterator.hasNext()) {
IIRow iiRow = iiRowIterator.next();
@@ -233,7 +239,7 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
throw new NotImplementedException();
}
@@ -243,7 +249,7 @@ public class IITest extends LocalFileMetadataTestCase {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1062749..9090068 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -645,5 +645,8 @@ abstract public class KylinConfigBase implements Serializable {
public int getCubeStatsHLLPrecision() {
return Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", "14"));
}
-
+
+ public String getPatchedFuzzyRowFilterVersion() {
+ return this.getOptional("kylin.hbase.filter.fuzzy.row.filter.version", "1.1.3");
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index bd947af..6082fa9 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -1,20 +1,18 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
-->
+
<configuration>
<property>
@@ -26,44 +24,41 @@
</description>
</property>
- <property>
- <name>mapreduce.map.maxattempts</name>
- <value>2</value>
- </property>
+ <!-- uncomment the following 5 properties to enable lzo compressing
- <!--
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
- <property>
- <name>mapred.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
+ <property>
+ <name>mapred.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
--->
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ !-->
<property>
<name>mapreduce.job.max.split.locations</name>
@@ -76,5 +71,4 @@
<value>2</value>
<description>Block replication</description>
</property>
-
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 18f6feb..ff1c7eb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -28,12 +28,12 @@
<property>
<name>mapred.job.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
<name>mapred.job.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -58,7 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,10 +81,9 @@
<value>false</value>
</property>
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
+ <value>0.05</value>
</property>
<property>
@@ -114,7 +113,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -124,7 +123,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -169,7 +168,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -219,7 +218,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -234,7 +233,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -244,7 +243,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 07de5d7..a83a754 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -37,8 +37,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
@@ -60,6 +59,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
@@ -404,34 +404,34 @@ public class BuildCubeWithEngine {
}
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- long totalSize = 0;
- for (Long size : sizeMap.values()) {
- totalSize += size;
- }
- if (totalSize == 0) {
- return;
- }
- Map<byte[], Pair<Integer, Integer>> countMap = cal.getRegionHFileCountMap();
- // check if there's region contains more than one hfile, which means the hfile config take effects
- boolean hasMultiHFileRegions = false;
- for (Pair<Integer, Integer> count : countMap.values()) {
- // check if hfile count is greater than store count
- if (count.getSecond() > count.getFirst()) {
- hasMultiHFileRegions = true;
- break;
- }
- }
- if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() == 0 && hasMultiHFileRegions) {
- throw new IOException("hfile size set to 0, but found region contains more than one hfiles");
- } else if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() > 0 && !hasMultiHFileRegions) {
- throw new IOException("hfile size set greater than 0, but all regions still has only one hfile");
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ long totalSize = 0;
+ for (Long size : sizeMap.values()) {
+ totalSize += size;
+ }
+ if (totalSize == 0) {
+ return;
+ }
+
+ Map<byte[], Pair<Integer, Integer>> countMap = cal.getRegionHFileCountMap();
+ // check if there's region contains more than one hfile, which means the hfile config take effects
+ boolean hasMultiHFileRegions = false;
+ for (Pair<Integer, Integer> count : countMap.values()) {
+ // check if hfile count is greater than store count
+ if (count.getSecond() > count.getFirst()) {
+ hasMultiHFileRegions = true;
+ break;
}
}
+ if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() == 0 && hasMultiHFileRegions) {
+ throw new IOException("hfile size set to 0, but found region contains more than one hfiles");
+ } else if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() > 0 && !hasMultiHFileRegions) {
+ throw new IOException("hfile size set greater than 0, but all regions still has only one hfile");
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index c0f6f63..a74ecb9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -45,7 +45,8 @@ import java.util.TimeZone;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.util.ToolRunner;
@@ -208,7 +209,7 @@ public class BuildIIWithStream {
}
}
final IISegment segment = createSegment(iiName);
- final HTableInterface htable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(segment.getStorageLocationIdentifier());
+ final Table htable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(segment.getStorageLocationIdentifier()));
String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
ToolRunner.run(new IICreateHTableJob(), args);
@@ -246,7 +247,7 @@ public class BuildIIWithStream {
}
}
- private void build(SliceBuilder sliceBuilder, StreamingBatch batch, HTableInterface htable) throws IOException {
+ private void build(SliceBuilder sliceBuilder, StreamingBatch batch, Table htable) throws IOException {
final Slice slice = sliceBuilder.buildSlice(batch);
try {
loadToHBase(htable, slice, new IIKeyValueCodec(slice.getInfo()));
@@ -255,17 +256,17 @@ public class BuildIIWithStream {
}
}
- private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
+ private void loadToHBase(Table hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
List<Put> data = Lists.newArrayList();
for (IIRow row : codec.encodeKeyValue(slice)) {
final byte[] key = row.getKey().get();
final byte[] value = row.getValue().get();
Put put = new Put(key);
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
+ put.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
final ImmutableBytesWritable dictionary = row.getDictionary();
final byte[] dictBytes = dictionary.get();
if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) {
- put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
+ put.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
} else {
throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
index b5be703..5a591a1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
@@ -20,9 +20,7 @@ package org.apache.kylin.storage.hbase.ii;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
@@ -49,7 +47,7 @@ public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
IIInstance ii;
IISegment seg;
- HConnection hconn;
+ Connection hconn;
TableRecordInfo info;
@@ -60,8 +58,7 @@ public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
this.seg = ii.getFirstSegment();
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- hconn = HConnectionManager.createConnection(hconf);
+ hconn = HBaseConnection.get(getTestConfig().getStorageUrl());
this.info = new TableRecordInfo(seg);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 12d9eb2..df29c0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -47,12 +47,12 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<zookeeper.version>3.4.6</zookeeper.version>
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.8.1</kafka.version>
<!-- Dependency versions -->
@@ -64,6 +64,7 @@
<!-- Commons -->
<commons-cli.version>1.2</commons-cli.version>
+ <commons-codec.version>1.4</commons-codec.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.1</commons-lang3.version>
<commons-collections.version>3.2.1</commons-collections.version>
@@ -103,7 +104,7 @@
<calcite.version>1.6.0</calcite.version>
<!-- Curator.version Version -->
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
@@ -324,6 +325,11 @@
<version>${commons-cli.version}</version>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons-codec.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 2220113..0e71cd5 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -37,6 +37,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index 193f5a8..933c49d 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.UserService;
@@ -29,8 +29,8 @@ import org.apache.kylin.rest.service.UserService;
*/
public class MockAclHBaseStorage implements AclHBaseStorage {
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
@@ -49,7 +49,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
return mockedAclTable;
} else if (StringUtils.equals(tableName, userTableName)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 5a48e83..d40bdf3 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.UserService;
@@ -56,11 +57,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/AclService.java b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
index 58e093c..a03ff5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -225,17 +225,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -249,7 +248,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -265,7 +264,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -283,7 +281,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -294,17 +292,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 8ed0802..6c47fde 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -20,10 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
import java.util.*;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.mapred.Merger;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -431,35 +428,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
-
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- if (null != table) {
- table.close();
- }
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 08b338c..6691784 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -44,8 +44,9 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
@@ -130,14 +131,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -163,14 +163,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -182,9 +181,9 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/server/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/UserService.java b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index e4e2de3..624c49c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -29,7 +29,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -69,7 +69,7 @@ public class UserService implements UserManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -100,16 +100,15 @@ public class UserService implements UserManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
byte[] userAuthorities = serialize(user.getAuthorities());
htable = aclHBaseStorage.getTable(userTableName);
Put put = new Put(Bytes.toBytes(user.getUsername()));
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), userAuthorities);
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), userAuthorities);
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -119,14 +118,13 @@ public class UserService implements UserManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -141,7 +139,7 @@ public class UserService implements UserManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -161,7 +159,7 @@ public class UserService implements UserManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<String> authorities = new ArrayList<String>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index e2f4f3a..a7ffa0d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -32,9 +32,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -52,13 +52,13 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> ConfigCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> ConnPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> ConnPool = new ConcurrentHashMap<String, Connection>();
static {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
- for (HConnection conn : ConnPool.values()) {
+ for (Connection conn : ConnPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -124,9 +124,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = ConfigCache.get(url);
if (conf == null) {
@@ -134,13 +134,13 @@ public class HBaseConnection {
ConfigCache.put(url, conf);
}
- HConnection connection = ConnPool.get(url);
+ Connection connection = ConnPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
ConnPool.put(url, connection);
}
@@ -159,8 +159,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -180,8 +180,8 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void createHTableIfNeeded(Connection conn, String tableName, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (tableExists(conn, tableName)) {
@@ -210,8 +210,8 @@ public class HBaseConnection {
}
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -221,10 +221,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 2262482..07a5586 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,10 +31,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- private HConnection getConnection() throws IOException {
+ private Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,11 +273,10 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -304,7 +301,7 @@ public class HBaseResourceStore extends ResourceStore {
get.addColumn(B_FAMILY, B_COLUMN_TS);
}
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
Result result = table.get(get);
boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
@@ -314,7 +311,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -340,7 +337,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -348,8 +345,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index bbdb542..81f205e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 9eb05d2..7104da8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,8 +25,9 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 47e94de..7211ec1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Pair;
@@ -147,7 +147,7 @@ public class CubeStorageQuery implements IStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
index 8aace22..aef3963 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
@@ -24,8 +24,9 @@ import java.util.Iterator;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -40,14 +41,14 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
byte[] family;
- HTableInterface table;
+ Table table;
ResultScanner scanner;
Iterator<Result> iterator;
- public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family) throws IOException {
+ public HBaseClientKVIterator(Connection hconn, String tableName, byte[] family) throws IOException {
this.family = family;
- this.table = hconn.getTable(tableName);
+ this.table = hconn.getTable(TableName.valueOf(tableName));
this.scanner = table.getScanner(family);
this.iterator = scanner.iterator();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 6342c5c..ae442fe 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -23,9 +23,11 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +52,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +62,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +96,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index c7b650a..8dba1b1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index d8b61b3..635a32f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -24,7 +24,9 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -116,8 +118,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +128,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +162,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..1809a44 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -24,12 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +110,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +120,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +159,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb740ec8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index f0e9bed..c69fd8b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,7 +23,7 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
[4/8] kylin git commit: KYLIN-1721 KylinConfigExt lost base
properties when store into file
Posted by li...@apache.org.
KYLIN-1721 KylinConfigExt lost base properties when store into file
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8c10e9ea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8c10e9ea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8c10e9ea
Branch: refs/heads/1.5.x-HBase1.x
Commit: 8c10e9eaaf8345d4ea9b6ae17878057d4dbab6f4
Parents: b24b7e6
Author: sunyerui <su...@gmail.com>
Authored: Sun May 22 01:17:13 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun May 22 13:26:47 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigExt.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8c10e9ea/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
index 4cd1339..c3b45d7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
@@ -62,7 +62,8 @@ public class KylinConfigExt extends KylinConfig {
}
protected Properties getAllProperties() {
- Properties result = new Properties(super.getAllProperties());
+ Properties result = new Properties();
+ result.putAll(super.getAllProperties());
result.putAll(overrides);
return result;
}
[3/8] kylin git commit: KYLIN-1323 a bug in hbase read-write
separation mode
Posted by li...@apache.org.
KYLIN-1323 a bug in hbase read-write separation mode
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b24b7e64
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b24b7e64
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b24b7e64
Branch: refs/heads/1.5.x-HBase1.x
Commit: b24b7e642e7c7cf06744f5329bc1aa55624b64e4
Parents: 60a7c74
Author: shaofengshi <sh...@apache.org>
Authored: Fri May 20 07:33:41 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri May 20 02:45:03 2016 +0000
----------------------------------------------------------------------
.../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b24b7e64/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index e1f8818..29bcb62 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -49,7 +48,6 @@ import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
import org.apache.kylin.engine.mr.common.CuboidShardUtil;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -282,7 +280,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
return;
}
- FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration());
if (fs.exists(outputFolder) == false) {
fs.mkdirs(outputFolder);
}
[6/8] kylin git commit: [maven-release-plugin] prepare release
kylin-1.5.2
Posted by li...@apache.org.
[maven-release-plugin] prepare release kylin-1.5.2
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08807fac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08807fac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08807fac
Branch: refs/heads/1.5.x-HBase1.x
Commit: 08807facb8889380de8da4515efe359d68d3187d
Parents: a372fe3
Author: lidongsjtu <li...@apache.org>
Authored: Sun May 22 08:32:38 2016 +0000
Committer: lidongsjtu <li...@apache.org>
Committed: Sun May 22 08:32:38 2016 +0000
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
engine-streaming/pom.xml | 2 +-
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml | 5 +++--
query/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
tool/pom.xml | 5 ++---
21 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 58f9e19..b6e3675 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index d6c57f3..7c70df3 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 0c85124..7647621 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index f41b507..2309872 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 17319e0..1af0091 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 09f20e3..864bf1b 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index cd4abf9..f65129d 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index b53bbc0..2f08dd9 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 8cc86d0..379eaa6 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index d0c5146..eda0513 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index dba8eb0..3613cfe 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 95ba45f..95fa443 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index a1879c4..3de7817 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 1a3efb7..dec032d 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 326ae30..12d9eb2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
<name>Kylin:HadoopOLAPEngine</name>
<url>http://kylin.apache.org</url>
@@ -169,7 +169,8 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/kylin.git</url>
- </scm>
+ <tag>kylin-1.5.2</tag>
+ </scm>
<dependencyManagement>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 4730fbc..17f8169 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 87c3eb0..36d7b83 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 796b9c1..2cb5581 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index a3f90bf..398409b 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 397410e..f067bf6 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
http://git-wip-us.apache.org/repos/asf/kylin/blob/08807fac/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 2ffa79e..d5b35f4 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -17,12 +17,11 @@
limitations under the License.
-->
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>1.5.2-SNAPSHOT</version>
+ <version>1.5.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>