You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/05 02:54:15 UTC
[01/31] incubator-kylin git commit: KYLIN-1101 add date format field
for partition column
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1112 6ab480ce7 -> 4f468d33c
KYLIN-1101 add date format field for partition column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/51495704
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/51495704
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/51495704
Branch: refs/heads/KYLIN-1112
Commit: 51495704e556858883fbc4a333da35e281ebb351
Parents: 25a210d
Author: jiazhong <ji...@ebay.com>
Authored: Thu Oct 29 17:22:58 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:15 2015 +0800
----------------------------------------------------------------------
webapp/app/js/model/cubeConfig.js | 6 +++++-
webapp/app/js/model/metaModel.js | 3 ++-
.../modelDesigner/conditions_settings.html | 19 +++++++++++++++++++
3 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/51495704/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 3d80c7f..c7e1b44 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -71,5 +71,9 @@ KylinApp.constant('cubeConfig', {
{attr: 'owner', name: 'Owner'},
{attr: 'create_time', name: 'Create Time'}
],
- streamingAutoGenerateMeasure:['year_start_ts','month_start_ts','day_start_ts','hour_start_ts','min_start_ts']
+ streamingAutoGenerateMeasure:['year_start_ts','month_start_ts','day_start_ts','hour_start_ts','min_start_ts'],
+ partitionDateFormatOpt:[
+ 'yyyy-MM-dd',
+ 'yyyyMMdd'
+ ]
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/51495704/webapp/app/js/model/metaModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/metaModel.js b/webapp/app/js/model/metaModel.js
index 3c70822..e2d93b2 100644
--- a/webapp/app/js/model/metaModel.js
+++ b/webapp/app/js/model/metaModel.js
@@ -87,7 +87,8 @@ KylinApp.service('MetaModel',function(){
"partition_desc" : {
"partition_date_column" : null,
"partition_date_start" : null,
- "partition_type" : 'APPEND'
+ "partition_type" : 'APPEND',
+ "partition_date_format":'yyyy-MM-dd'
},
last_modified:0
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/51495704/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 6e836ee..6ee805c 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -73,6 +73,25 @@
</div>
</div>
+
+ <!--Date Format-->
+ <div class="form-group">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select style="width: 100%" chosen
+ ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
+ ng-if="state.mode=='edit'"
+ data-placement=""
+ ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
+ <option value="">--Select Date Format--</option>
+ </select>
+ <span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_date_column">{{(modelsManager.selectedModel.partition_desc.partition_date_format)}}</span>
+ </div>
+ </div>
+ </div>
+
<div class="form-group" ng-show="userService.hasRole('ROLE_ADMIN')">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Cube Size</b></label>
[18/31] incubator-kylin git commit: KYLIN-1058 Remove right join
during model creation
Posted by sh...@apache.org.
KYLIN-1058 Remove right join during model creation
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b4c53ad0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b4c53ad0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b4c53ad0
Branch: refs/heads/KYLIN-1112
Commit: b4c53ad0936be3470f35273562c2035fb1802b6c
Parents: 107d339
Author: jiazhong <ji...@ebay.com>
Authored: Fri Oct 30 14:43:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
webapp/app/js/model/cubeConfig.js | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b4c53ad0/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index c7e1b44..a647b53 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -27,8 +27,7 @@ KylinApp.constant('cubeConfig', {
cubePartitionTypes: ['APPEND'],
joinTypes: [
{name: 'Left', value: 'left'},
- {name: 'Inner', value: 'inner'},
- {name: 'Right', value: 'right'}
+ {name: 'Inner', value: 'inner'}
],
queryPriorities: [
{name: 'NORMAL', value: 50},
[26/31] incubator-kylin git commit: KYLIN-1078 Cannot have comments
in the end of New Query textbox
Posted by sh...@apache.org.
KYLIN-1078 Cannot have comments in the end of New Query textbox
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bb3aa4fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bb3aa4fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bb3aa4fb
Branch: refs/heads/KYLIN-1112
Commit: bb3aa4fbd3094595720fabbe0307e6de98747670
Parents: e6246e1
Author: lidongsjtu <do...@ebay.com>
Authored: Wed Nov 4 16:17:01 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bb3aa4fb/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 9b5e925..a08d2bd 100644
--- a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -107,12 +107,12 @@ public class QueryUtil {
int limit = sqlRequest.getLimit();
if (limit > 0 && !sql.toLowerCase().contains("limit")) {
- sql += (" LIMIT " + limit);
+ sql += ("\nLIMIT " + limit);
}
int offset = sqlRequest.getOffset();
if (offset > 0 && !sql.toLowerCase().contains("offset")) {
- sql += (" OFFSET " + offset);
+ sql += ("\nOFFSET " + offset);
}
return healSickSql(sql);
[19/31] incubator-kylin git commit: KYLIN-946 check project when
refresh query page
Posted by sh...@apache.org.
KYLIN-946 check project when refresh query page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7148a178
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7148a178
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7148a178
Branch: refs/heads/KYLIN-1112
Commit: 7148a178076009f0f4d122484e279d953af47b8f
Parents: 33b7b85
Author: jiazhong <ji...@ebay.com>
Authored: Mon Nov 2 14:43:14 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/projectMeta.js | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7148a178/webapp/app/js/controllers/projectMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/projectMeta.js b/webapp/app/js/controllers/projectMeta.js
index d0e8781..174597e 100644
--- a/webapp/app/js/controllers/projectMeta.js
+++ b/webapp/app/js/controllers/projectMeta.js
@@ -35,8 +35,11 @@ KylinApp
$scope.projectMetaLoad = function () {
var defer = $q.defer();
$scope.selectedSrcDb = [];
+ if(!$scope.projectModel.getSelectedProject()) {
+ return;
+ }
$scope.loading = true;
- QueryService.getTables({project: $scope.projectModel.selectedProject}, {}, function (tables) {
+ QueryService.getTables({project: $scope.projectModel.getSelectedProject()}, {}, function (tables) {
var tableMap = [];
angular.forEach(tables, function (table) {
if (!tableMap[table.table_SCHEM]) {
[08/31] incubator-kylin git commit: KYLIN-942 correct math3 dependency
Posted by sh...@apache.org.
KYLIN-942 correct math3 dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/07038d9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/07038d9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/07038d9a
Branch: refs/heads/KYLIN-1112
Commit: 07038d9a06532c03c25799211582d3c204ede283
Parents: 6b831ff
Author: honma <ho...@ebay.com>
Authored: Mon Nov 2 16:02:55 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
core-common/pom.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/07038d9a/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 3275532..d02ddd3 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -49,6 +49,10 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
[11/31] incubator-kylin git commit: KYLIN-1113 Support TopN query in
v2/CubeStorageQuery.java
Posted by sh...@apache.org.
KYLIN-1113 Support TopN query in v2/CubeStorageQuery.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/65fb7213
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/65fb7213
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/65fb7213
Branch: refs/heads/KYLIN-1112
Commit: 65fb72137f4b1fdd9b1a7244b4fb59fe1bec7636
Parents: b0e3096
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 29 14:41:46 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 5 +++++
.../apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java | 3 ++-
.../storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java | 1 +
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/65fb7213/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 8317403..4ced852 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -144,6 +144,11 @@ public class CubeStorageQuery implements ICachableStorageQuery {
if (sqlDigest.metricColumns.contains(column)) {
continue;
}
+
+ // skip topN display col
+ if (topNColumns.contains(column)) {
+ continue;
+ }
dimensions.add(column);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/65fb7213/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index 2354a88..c89cce2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -107,7 +107,8 @@ public class CubeTupleConverter {
if (this.topNCol != null) {
this.topNColTupleIdx = tupleInfo.hasColumn(this.topNCol) ? tupleInfo.getColumnIndex(this.topNCol) : -1;
- this.topNMeasureTupleIdx = nSelectedDims;
+ // topN only allow 1 measure
+ this.topNMeasureTupleIdx = tupleIdx[tupleIdx.length - 1];
this.topNColDict = (Dictionary<String>)cubeSeg.getDictionary(this.topNCol);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/65fb7213/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
index aa50d00..f162be6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
@@ -23,6 +23,7 @@ public class SequentialCubeTopNTupleIterator extends SequentialCubeTupleIterator
TblColRef topNCol, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
super(scanners, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context);
+ this.topNCol = topNCol;
}
@Override
[12/31] incubator-kylin git commit: KYLIN-1068 support topn in v2
CubeStorageQuery
Posted by sh...@apache.org.
KYLIN-1068 support topn in v2 CubeStorageQuery
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b0e30966
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b0e30966
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b0e30966
Branch: refs/heads/KYLIN-1112
Commit: b0e3096662487f1114a8dda49c70ea70d75ec649
Parents: 279226f
Author: shaofengshi <sh...@apache.org>
Authored: Wed Oct 28 21:39:13 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query82.sql | 27 ++++++++
.../resources/query/sql/query82.sql.disable | 27 --------
.../storage/hbase/cube/v2/CubeStorageQuery.java | 62 ++++++++++++++++++
.../hbase/cube/v2/CubeTupleConverter.java | 61 +++++++++++++++++-
.../v2/SequentialCubeTopNTupleIterator.java | 67 ++++++++++++++++++++
.../cube/v2/SequentialCubeTupleIterator.java | 28 ++++----
6 files changed, 230 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/query/src/test/resources/query/sql/query82.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql b/query/src/test/resources/query/sql/query82.sql
new file mode 100644
index 0000000..57e9de0
--- /dev/null
+++ b/query/src/test/resources/query/sql/query82.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.cal_dt, seller_id
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ group by
+ test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/query/src/test/resources/query/sql/query82.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql.disable b/query/src/test/resources/query/sql/query82.sql.disable
deleted file mode 100644
index 57e9de0..0000000
--- a/query/src/test/resources/query/sql/query82.sql.disable
+++ /dev/null
@@ -1,27 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.cal_dt, seller_id
- FROM test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- group by
- test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 1d9ee3a..8317403 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -46,15 +46,30 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
+ private Collection<TblColRef> topNColumns;
public CubeStorageQuery(CubeInstance cube) {
this.cubeInstance = cube;
this.cubeDesc = cube.getDescriptor();
+ this.topNColumns = Lists.newArrayList();
+ for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
+ if (measureDesc.getFunction().isTopN()) {
+ List<TblColRef> colRefs = measureDesc.getFunction().getParameter().getColRefs();
+ topNColumns.add(colRefs.get(colRefs.size() - 1));
+ }
+ }
}
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ // check whether this is a TopN query;
+ checkAndRewriteTopN(context, sqlDigest, returnTupleInfo);
+
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
+ TblColRef topNCol = extractTopNCol(groups);
+ if (topNCol != null)
+ groups.remove(topNCol);
+
TupleFilter filter = sqlDigest.filter;
// build dimension & metrics
@@ -110,6 +125,9 @@ public class CubeStorageQuery implements ICachableStorageQuery {
if (scanners.isEmpty())
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+ if (topNCol != null)
+ return new SequentialCubeTopNTupleIterator(scanners, cuboid, dimensionsD, topNCol, metrics, returnTupleInfo, context);
+
return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
}
@@ -375,4 +393,48 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return false;
}
+
+ private void checkAndRewriteTopN(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ Collection<TblColRef> groups = sqlDigest.groupbyColumns;
+ TblColRef topNDisplayCol = extractTopNCol(groups);
+ boolean hasTopN = topNDisplayCol != null;
+
+ if (hasTopN == false)
+ return;
+
+ if (sqlDigest.aggregations.size() != 1) {
+ throw new IllegalStateException("When query with topN, only one metrics is allowed.");
+ }
+
+ FunctionDesc functionDesc = sqlDigest.aggregations.iterator().next();
+ if (functionDesc.isSum() == false) {
+ throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ }
+
+ FunctionDesc rewriteFunction = null;
+ // replace the SUM to the TopN function
+ for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
+ if (measureDesc.getFunction().isCompatible(functionDesc) && topNDisplayCol.getName().equalsIgnoreCase(measureDesc.getFunction().getParameter().getDisplayColumn())) {
+ rewriteFunction = measureDesc.getFunction();
+ break;
+ }
+ }
+
+ if (rewriteFunction == null) {
+ throw new IllegalStateException("Didn't find topN measure for " + functionDesc);
+ }
+
+ sqlDigest.aggregations = Lists.newArrayList(rewriteFunction);
+ logger.info("Rewrite function " + functionDesc + " to " + rewriteFunction);
+ }
+
+ private TblColRef extractTopNCol(Collection<TblColRef> colRefs) {
+ for (TblColRef colRef : colRefs) {
+ if (topNColumns.contains(colRef)) {
+ return colRef;
+ }
+ }
+
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index fb54cce..2354a88 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -18,17 +18,24 @@
package org.apache.kylin.storage.hbase.cube.v2;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.kylin.common.topn.Counter;
+import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -52,13 +59,18 @@ public class CubeTupleConverter {
final int[] tupleIdx;
final Object[] tmpValues;
final int nSelectedDims;
+ final TblColRef topNCol;
+ int topNColTupleIdx;
+ int topNMeasureTupleIdx;
+ Dictionary<String> topNColDict;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
+ Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, TblColRef topNCol) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
this.tupleInfo = returnTupleInfo;
this.derivedColFillers = Lists.newArrayList();
+ this.topNCol = topNCol;
List<TblColRef> cuboidDims = cuboid.getColumns();
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
@@ -93,6 +105,13 @@ public class CubeTupleConverter {
iii++;
}
+ if (this.topNCol != null) {
+ this.topNColTupleIdx = tupleInfo.hasColumn(this.topNCol) ? tupleInfo.getColumnIndex(this.topNCol) : -1;
+ this.topNMeasureTupleIdx = nSelectedDims;
+
+ this.topNColDict = (Dictionary<String>)cubeSeg.getDictionary(this.topNCol);
+ }
+
// prepare derived columns and filler
Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCube().getHostToDerivedInfo(cuboidDims, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
@@ -132,6 +151,12 @@ public class CubeTupleConverter {
}
}
+ public Iterator<Tuple> translateTopNResult(GTRecord record, Tuple tuple) {
+ translateResult(record, tuple);
+ Object topNCounterObj = tuple.getAllValues()[topNMeasureTupleIdx];
+ assert (topNCounterObj instanceof TopNCounter);
+ return new TopNCounterTupleIterator(tuple, (TopNCounter) topNCounterObj);
+ }
private interface IDerivedColumnFiller {
public void fillDerivedColumns(Object[] tmpValues, Tuple tuple);
}
@@ -221,4 +246,38 @@ public class CubeTupleConverter {
private static String toString(Object o) {
return o == null ? null : o.toString();
}
+
+ private class TopNCounterTupleIterator implements Iterator {
+
+ private Tuple tuple;
+ private Iterator<Counter> topNCounterIterator;
+ private Counter<ByteArray> counter;
+
+ private TopNCounterTupleIterator(Tuple tuple, TopNCounter topNCounter) {
+ this.tuple = tuple;
+ this.topNCounterIterator = topNCounter.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return topNCounterIterator.hasNext();
+ }
+
+ @Override
+ public Tuple next() {
+ counter = topNCounterIterator.next();
+ int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
+ String colValue = topNColDict.getValueFromId(key);
+ tuple.setDimensionValue(topNColTupleIdx, colValue);
+ tuple.setMeasureValue(topNMeasureTupleIdx, counter.getCount());
+
+ return tuple;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
new file mode 100644
index 0000000..aa50d00
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
@@ -0,0 +1,67 @@
+package org.apache.kylin.storage.hbase.cube.v2;
+
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Created by shaoshi on 10/28/15.
+ */
+public class SequentialCubeTopNTupleIterator extends SequentialCubeTupleIterator {
+
+ private Iterator<Tuple> innerResultIterator;
+ private TblColRef topNCol;
+
+ public SequentialCubeTopNTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
+ TblColRef topNCol, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
+
+ super(scanners, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context);
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (next != null)
+ return true;
+ if (innerResultIterator == null) {
+ if (curScanner == null) {
+ if (scannerIterator.hasNext()) {
+ curScanner = scannerIterator.next();
+ curRecordIterator = curScanner.iterator();
+ curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo, topNCol);
+ } else {
+ return false;
+ }
+ }
+
+ if (curRecordIterator.hasNext()) {
+ innerResultIterator = curTupleConverter.translateTopNResult(curRecordIterator.next(), tuple);
+ return hasNext();
+ } else {
+ close(curScanner);
+ curScanner = null;
+ curRecordIterator = null;
+ curTupleConverter = null;
+ innerResultIterator = null;
+ return hasNext();
+ }
+
+ }
+
+
+ if (innerResultIterator.hasNext()) {
+ next = innerResultIterator.next();
+ return true;
+ } else {
+ innerResultIterator = null;
+ return hasNext();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0e30966/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index 85aa54a..ddaafd5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -22,18 +22,18 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
private static final Logger logger = LoggerFactory.getLogger(SequentialCubeTupleIterator.class);
- private final Cuboid cuboid;
- private final Set<TblColRef> selectedDimensions;
- private final Set<FunctionDesc> selectedMetrics;
- private final TupleInfo tupleInfo;
- private final Tuple tuple;
- private final Iterator<CubeSegmentScanner> scannerIterator;
- private final StorageContext context;
-
- private CubeSegmentScanner curScanner;
- private Iterator<GTRecord> curRecordIterator;
- private CubeTupleConverter curTupleConverter;
- private Tuple next;
+ protected final Cuboid cuboid;
+ protected final Set<TblColRef> selectedDimensions;
+ protected final Set<FunctionDesc> selectedMetrics;
+ protected final TupleInfo tupleInfo;
+ protected final Tuple tuple;
+ protected final Iterator<CubeSegmentScanner> scannerIterator;
+ protected final StorageContext context;
+
+ protected CubeSegmentScanner curScanner;
+ protected Iterator<GTRecord> curRecordIterator;
+ protected CubeTupleConverter curTupleConverter;
+ protected Tuple next;
private int scanCount;
private int scanCountDelta;
@@ -58,7 +58,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
if (scannerIterator.hasNext()) {
curScanner = scannerIterator.next();
curRecordIterator = curScanner.iterator();
- curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+ curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo, null);
} else {
return false;
}
@@ -112,7 +112,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
}
}
- private void close(CubeSegmentScanner scanner) {
+ protected void close(CubeSegmentScanner scanner) {
try {
scanner.close();
} catch (IOException e) {
[22/31] incubator-kylin git commit: enable ITIIQueryTest
Posted by sh...@apache.org.
enable ITIIQueryTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4f468d33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4f468d33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4f468d33
Branch: refs/heads/KYLIN-1112
Commit: 4f468d33c7843d046d8a9f43764c6b5ac6d96e95
Parents: b7cb41a
Author: shaofengshi <sh...@apache.org>
Authored: Mon Nov 2 15:56:22 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f468d33/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
index 8aace22..b18d258 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
@@ -82,7 +82,8 @@ public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES);
- dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ if (c != null)
+ dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
return pair;
}
[02/31] incubator-kylin git commit: KYLIN-1036 remove unused png file
Posted by sh...@apache.org.
KYLIN-1036 remove unused png file
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/25a210df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/25a210df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/25a210df
Branch: refs/heads/KYLIN-1112
Commit: 25a210df6647a57393a48fd08939baa79fbbbfa2
Parents: a1ed71f
Author: jiazhong <ji...@ebay.com>
Authored: Thu Oct 29 15:19:02 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:15 2015 +0800
----------------------------------------------------------------------
webapp/app/image/cube1.png | Bin 24538 -> 0 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25a210df/webapp/app/image/cube1.png
----------------------------------------------------------------------
diff --git a/webapp/app/image/cube1.png b/webapp/app/image/cube1.png
deleted file mode 100644
index 1950764..0000000
Binary files a/webapp/app/image/cube1.png and /dev/null differ
[13/31] incubator-kylin git commit: KYLIN-1113 enable query83.sql
Posted by sh...@apache.org.
KYLIN-1113 enable query83.sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/88d6485c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/88d6485c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/88d6485c
Branch: refs/heads/KYLIN-1112
Commit: 88d6485c8793a6fe6f8c71dc057b12168817409c
Parents: 65fb721
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 29 14:43:32 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query83.sql | 29 ++++++++++++++++++++
.../resources/query/sql/query83.sql.disable | 29 --------------------
2 files changed, 29 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/88d6485c/query/src/test/resources/query/sql/query83.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql b/query/src/test/resources/query/sql/query83.sql
new file mode 100644
index 0000000..514beb4
--- /dev/null
+++ b/query/src/test/resources/query/sql/query83.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ seller_id
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ where test_kylin_fact.cal_dt < DATE '2013-02-01'
+ group by
+ test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/88d6485c/query/src/test/resources/query/sql/query83.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql.disable b/query/src/test/resources/query/sql/query83.sql.disable
deleted file mode 100644
index 514beb4..0000000
--- a/query/src/test/resources/query/sql/query83.sql.disable
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT
- seller_id
- FROM test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- where test_kylin_fact.cal_dt < DATE '2013-02-01'
- group by
- test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
[07/31] incubator-kylin git commit: KYLIN-942 bug fixs and minor
changes on configurations
Posted by sh...@apache.org.
KYLIN-942 bug fixs and minor changes on configurations
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7ced7f67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7ced7f67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7ced7f67
Branch: refs/heads/KYLIN-1112
Commit: 7ced7f67c99009f7056d98bd0f240820ecc72dcd
Parents: 5fec41f
Author: honma <ho...@ebay.com>
Authored: Fri Oct 30 15:34:20 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 6 +++---
core-common/pom.xml | 6 ------
dev-support/test_all.sh | 11 -----------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 11 +++++++++++
.../org/apache/kylin/query/test/ITKylinQueryTest.java | 2 +-
query/src/test/resources/query/temp/.gitignore | 1 +
.../cube/v2/coprocessor/endpoint/CubeVisitService.java | 1 +
7 files changed, 17 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index fe953fc..cc91824 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -56,9 +56,9 @@ kylin.job.cubing.inMem.sampling.percent=100
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
-kylin.hbase.region.cut.small=10
-kylin.hbase.region.cut.medium=20
-kylin.hbase.region.cut.large=100
+kylin.hbase.region.cut.small=5
+kylin.hbase.region.cut.medium=10
+kylin.hbase.region.cut.large=50
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index f7444be..3275532 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -89,7 +89,6 @@
<groupId>com.ning</groupId>
<artifactId>compress-lzf</artifactId>
</dependency>
-
<!-- Logging -->
<dependency>
@@ -115,11 +114,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/dev-support/test_all.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all.sh b/dev-support/test_all.sh
deleted file mode 100644
index 6a7b887..0000000
--- a/dev-support/test_all.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests | tee mci.log
-mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithEngineTest.log
-mvn test -Dtest=org.apache.kylin.job.BuildIIWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildIIWithStreamTest.log
-mvn test -Dtest=org.apache.kylin.job.BuildCubeWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithStreamTest.log
-mvn verify -fae -P sandbox | tee mvnverify.log
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100644
index 0000000..1f63f71
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests | tee mci.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithEngineTest.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildIIWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildIIWithStreamTest.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildCubeWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithStreamTest.log
+mvn verify -Dhdp.version=2.2.4.2-2 -fae -P sandbox | tee mvnverify.log
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
index 4821ce9..72c366b 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -95,7 +95,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql/query82.sql";
+ String queryFileName = "src/test/resources/query/temp/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/query/src/test/resources/query/temp/.gitignore
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/temp/.gitignore b/query/src/test/resources/query/temp/.gitignore
new file mode 100644
index 0000000..d1b811b
--- /dev/null
+++ b/query/src/test/resources/query/temp/.gitignore
@@ -0,0 +1 @@
+*.sql
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7ced7f67/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index ba766bd..fb497b9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -56,6 +56,7 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
+@SuppressWarnings("unused")//used in hbase endpoint
public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);
[24/31] incubator-kylin git commit: KYLIN-1121 Fix
newHBaseConfiguration(url) to fail on bad URL input
Posted by sh...@apache.org.
KYLIN-1121 Fix newHBaseConfiguration(url) to fail on bad URL input
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1edbf478
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1edbf478
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1edbf478
Branch: refs/heads/KYLIN-1112
Commit: 1edbf47844c41db689ba79efbb71248e5879aedd
Parents: 276dfbe
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Nov 3 18:15:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/storage/hbase/HBaseConnection.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1edbf478/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index abbbc01..d1a92a0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -80,7 +80,8 @@ public class HBaseConnection {
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
- assert (StringUtils.isEmpty(url) || "hbase".equals(url)) : "for hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties";
+ if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
+ throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
// support hbase using a different FS
String hbaseClusterFs = KylinConfig.getInstanceFromEnv().getHBaseClusterFs();
[15/31] incubator-kylin git commit: KYLIN-1067 Kylin get MapReduce
Job status with "yarn.resourcemanager.webapp.address" need suport
ResourceManager HA config
Posted by sh...@apache.org.
KYLIN-1067 Kylin get MapReduce Job status with "yarn.resourcemanager.webapp.address" need suport ResourceManager HA config
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/933e81fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/933e81fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/933e81fd
Branch: refs/heads/KYLIN-1112
Commit: 933e81fd17cb29893ca13e8245598c9d20b6fa94
Parents: b4c53ad
Author: wangxiaoyu8 <wa...@jd.com>
Authored: Mon Oct 12 11:34:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/MapReduceExecutable.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/933e81fd/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index c1aa42e..4fb3eed 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -32,6 +32,9 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.RMHAUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -198,7 +201,12 @@ public class MapReduceExecutable extends AbstractExecutable {
} else {
logger.info(KylinConfig.KYLIN_JOB_YARN_APP_REST_CHECK_URL + " is not set, read from job configuration");
}
- String rmWebHost = job.getConfiguration().get("yarn.resourcemanager.webapp.address");
+ String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, job.getConfiguration());
+ if(HAUtil.isHAEnabled(job.getConfiguration())) {
+ YarnConfiguration conf = new YarnConfiguration(job.getConfiguration());
+ String active = RMHAUtils.findActiveRMHAId(conf);
+ rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
+ }
if (StringUtils.isEmpty(rmWebHost)) {
return null;
}
[03/31] incubator-kylin git commit: KYLIN-1108 always return BIGINT
type for COUNT expression
Posted by sh...@apache.org.
KYLIN-1108 always return BIGINT type for COUNT expression
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a1ed71fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a1ed71fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a1ed71fc
Branch: refs/heads/KYLIN-1112
Commit: a1ed71fca63f78ef474f4f947db9fdd23e931bac
Parents: b9e64da
Author: jiazhong <ji...@ebay.com>
Authored: Thu Oct 29 15:02:39 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:15 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1ed71fc/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index cdcd3cf..1560d23 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -44,6 +44,11 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
//map right return type for param
$scope.measureReturnTypeUpdate = function(){
+
+ if($scope.newMeasure.function.expression == 'COUNT'){
+ $scope.newMeasure.function.parameter.type= 'constant';
+ }
+
if($scope.newMeasure.function.parameter.type=="constant"&&$scope.newMeasure.function.expression!=="COUNT_DISTINCT"){
switch($scope.newMeasure.function.expression){
case "SUM":
[21/31] incubator-kylin git commit: KYLIN-1115 Clean up ODBC driver
code
Posted by sh...@apache.org.
KYLIN-1115 Clean up ODBC driver code
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4e161aac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4e161aac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4e161aac
Branch: refs/heads/KYLIN-1112
Commit: 4e161aac64a8947e88246ef78f1f29a679193fe8
Parents: 584f24e
Author: lidongsjtu <do...@ebay.com>
Authored: Fri Oct 30 17:05:45 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
odbc/.gitignore | 13 +-
odbc/Common/Common.vcxproj | 18 ++
odbc/Common/Common.vcxproj.filters | 19 ++
odbc/Common/Common.vcxproj.user | 19 ++
odbc/Common/Gzip.cpp | 1 -
odbc/Common/ReadMe.txt | 18 ++
odbc/Driver/GODBC.RC | 18 ++
odbc/Driver/driver.DEF | 17 ++
odbc/Driver/driver.dsp | 247 -------------------
odbc/Driver/driver.vcxproj | 19 ++
odbc/Driver/driver.vcxproj.filters | 19 ++
odbc/Driver/driver.vcxproj.user | 19 ++
odbc/Driver/version.props | 19 ++
odbc/Installer(64bit)/Installer(64bit).isl | 12 +-
odbc/Installer(64bit)/Installer(64bit).isproj | 17 ++
.../Installer(64bit)/Express/Interm/.gitignore | 1 -
.../Express/SingleImage/LogFiles/.gitignore | 1 -
.../Express/SingleImage/Reports/.gitignore | 1 -
odbc/Installer/Installer.isl | 12 +-
odbc/Installer/Installer.isproj | 17 ++
.../Installer/Express/Interm/.gitignore | 1 -
.../Express/SingleImage/LogFiles/.gitignore | 1 -
.../Express/SingleImage/Reports/.gitignore | 1 -
odbc/KylinODBC.sln | 18 +-
odbc/README.md | 2 +-
odbc/TestDLL/ColorPrint.cpp | 2 +-
odbc/TestDLL/ColorPrint.h | 2 +-
odbc/TestDLL/CompareQueryTests.cpp | 66 ++++-
odbc/TestDLL/Report.cpp | 2 +-
odbc/TestDLL/RestAPITest.cpp | 2 +-
odbc/TestDLL/SimpleQueryTest.cpp | 2 +-
odbc/TestDLL/Source.cpp | 5 +-
odbc/TestDLL/TestDLL.vcxproj | 20 ++
odbc/TestDLL/TestDLL.vcxproj.filters | 19 ++
odbc/TestDLL/TestDLL.vcxproj.user | 19 ++
odbc/TestDLL/Tests.h | 3 +-
odbc/TestDLL/testqueries.txt | 22 +-
odbc/doc/reference/ODBC Driver Development.docx | Bin 238372 -> 0 bytes
... driver based on REST service (chinese).docx | Bin 681289 -> 0 bytes
39 files changed, 409 insertions(+), 285 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/.gitignore b/odbc/.gitignore
index a8209c6..2af835b 100644
--- a/odbc/.gitignore
+++ b/odbc/.gitignore
@@ -3,6 +3,8 @@
*.lo
*.o
*.obj
+*.idb
+*.ilk
# Precompiled Headers
*.gch
@@ -22,23 +24,26 @@
*.a
*.lib
-*.exe
-
# Executables
*.out
*.app
+*.exe
+# Temp files
*.opensdf
*.sdf
*.log
*.tlog
*.pdb
*.suo
-
-
+*.aps
/UpgradeLog.*
/_UpgradeReport_files*
/ipch*
*.unsuccessfulbuild
*.successfulbuild
*.lastbuildstate
+
+# Installer projects
+Installer/Installer/
+Installer(64bit)/Installer(64bit)/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Common/Common.vcxproj
----------------------------------------------------------------------
diff --git a/odbc/Common/Common.vcxproj b/odbc/Common/Common.vcxproj
index a133099..a9f5cc6 100644
--- a/odbc/Common/Common.vcxproj
+++ b/odbc/Common/Common.vcxproj
@@ -1,4 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Common/Common.vcxproj.filters
----------------------------------------------------------------------
diff --git a/odbc/Common/Common.vcxproj.filters b/odbc/Common/Common.vcxproj.filters
index 790f9df..e729e00 100644
--- a/odbc/Common/Common.vcxproj.filters
+++ b/odbc/Common/Common.vcxproj.filters
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Common/Common.vcxproj.user
----------------------------------------------------------------------
diff --git a/odbc/Common/Common.vcxproj.user b/odbc/Common/Common.vcxproj.user
index a375ae3..27d8ac0 100644
--- a/odbc/Common/Common.vcxproj.user
+++ b/odbc/Common/Common.vcxproj.user
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Common/Gzip.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/Gzip.cpp b/odbc/Common/Gzip.cpp
index bbfc00c..220a1b8 100644
--- a/odbc/Common/Gzip.cpp
+++ b/odbc/Common/Gzip.cpp
@@ -1,4 +1,3 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Common/ReadMe.txt
----------------------------------------------------------------------
diff --git a/odbc/Common/ReadMe.txt b/odbc/Common/ReadMe.txt
index 514fbd1..aa14f76 100644
--- a/odbc/Common/ReadMe.txt
+++ b/odbc/Common/ReadMe.txt
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
========================================================================
STATIC LIBRARY : Common Project Overview
========================================================================
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/GODBC.RC
----------------------------------------------------------------------
diff --git a/odbc/Driver/GODBC.RC b/odbc/Driver/GODBC.RC
index 4a14c42..dd2cb3f 100644
--- a/odbc/Driver/GODBC.RC
+++ b/odbc/Driver/GODBC.RC
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/driver.DEF
----------------------------------------------------------------------
diff --git a/odbc/Driver/driver.DEF b/odbc/Driver/driver.DEF
index f1cf975..d0cdcdd 100644
--- a/odbc/Driver/driver.DEF
+++ b/odbc/Driver/driver.DEF
@@ -1,3 +1,20 @@
+;
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
LIBRARY driver
EXPORTS
SQLAllocConnect
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/driver.dsp
----------------------------------------------------------------------
diff --git a/odbc/Driver/driver.dsp b/odbc/Driver/driver.dsp
deleted file mode 100644
index 3849235..0000000
--- a/odbc/Driver/driver.dsp
+++ /dev/null
@@ -1,247 +0,0 @@
-# Microsoft Developer Studio Project File - Name="driver" - Package Owner=<4>
-# Microsoft Developer Studio Generated Build File, Format Version 6.00
-# ** DO NOT EDIT **
-
-# TARGTYPE "Win32 (x86) Dynamic-Link Library" 0x0102
-
-CFG=driver - Win32 Debug
-!MESSAGE This is not a valid makefile. To build this project using NMAKE,
-!MESSAGE use the Export Makefile command and run
-!MESSAGE
-!MESSAGE NMAKE /f "driver.mak".
-!MESSAGE
-!MESSAGE You can specify a configuration when running NMAKE
-!MESSAGE by defining the macro CFG on the command line. For example:
-!MESSAGE
-!MESSAGE NMAKE /f "driver.mak" CFG="driver - Win32 Debug"
-!MESSAGE
-!MESSAGE Possible choices for configuration are:
-!MESSAGE
-!MESSAGE "driver - Win32 Release" (based on "Win32 (x86) Dynamic-Link Library")
-!MESSAGE "driver - Win32 Debug" (based on "Win32 (x86) Dynamic-Link Library")
-!MESSAGE
-
-# Begin Project
-# PROP AllowPerConfigDependencies 0
-# PROP Scc_ProjName ""
-# PROP Scc_LocalPath ""
-CPP=cl.exe
-MTL=midl.exe
-RSC=rc.exe
-
-!IF "$(CFG)" == "driver - Win32 Release"
-
-# PROP BASE Use_MFC 0
-# PROP BASE Use_Debug_Libraries 0
-# PROP BASE Output_Dir "Release"
-# PROP BASE Intermediate_Dir "Release"
-# PROP BASE Target_Dir ""
-# PROP Use_MFC 0
-# PROP Use_Debug_Libraries 0
-# PROP Output_Dir "Release"
-# PROP Intermediate_Dir "Release"
-# PROP Ignore_Export_Lib 0
-# PROP Target_Dir ""
-# ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "driver_EXPORTS" /YX /FD /c
-# ADD CPP /nologo /MT /W3 /GX /O2 /I "." /I "..\common" /I "..\common\xml" /I "..\common\sock" /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "driver_EXPORTS" /D "_XML_STREAM_SOCK_CLIENT" /YX /FD /c
-# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32
-# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32
-# ADD BASE RSC /l 0x409 /d "NDEBUG"
-# ADD RSC /l 0x409 /d "NDEBUG"
-BSC32=bscmake.exe
-# ADD BASE BSC32 /nologo
-# ADD BSC32 /nologo
-LINK32=link.exe
-# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386
-# ADD LINK32 odbccp32.lib kernel32.lib user32.lib gdi32.lib advapi32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib mswsock.lib ws2_32.lib /nologo /dll /machine:I386
-
-!ELSEIF "$(CFG)" == "driver - Win32 Debug"
-
-# PROP BASE Use_MFC 0
-# PROP BASE Use_Debug_Libraries 1
-# PROP BASE Output_Dir "Debug"
-# PROP BASE Intermediate_Dir "Debug"
-# PROP BASE Target_Dir ""
-# PROP Use_MFC 0
-# PROP Use_Debug_Libraries 1
-# PROP Output_Dir ""
-# PROP Intermediate_Dir "Debug"
-# PROP Ignore_Export_Lib 0
-# PROP Target_Dir ""
-# ADD BASE CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "driver_EXPORTS" /YX /FD /GZ /c
-# ADD CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /I "." /I "..\common" /I "..\common\xml" /I "..\common\sock" /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "driver_EXPORTS" /D "_XML_STREAM_SOCK_CLIENT" /YX /FD /GZ /c
-# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32
-# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32
-# ADD BASE RSC /l 0x409 /d "_DEBUG"
-# ADD RSC /l 0x409 /d "_DEBUG"
-BSC32=bscmake.exe
-# ADD BASE BSC32 /nologo
-# ADD BSC32 /nologo
-LINK32=link.exe
-# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib advapi32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib mswsock.lib ws2_32.lib /nologo /dll /incremental:no /debug /machine:I386 /pdbtype:sept
-
-!ENDIF
-
-# Begin Target
-
-# Name "driver - Win32 Release"
-# Name "driver - Win32 Debug"
-# Begin Group "Source Files"
-
-# PROP Default_Filter "cpp;c;cxx;rc;def;r;odl;idl;hpj;bat"
-# Begin Source File
-
-SOURCE=..\Common\common.cpp
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_ALLOC.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_ATTR.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_CONN.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_CTLG.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_DESC.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_DIAG.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_DTYPE.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_EXEC.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_FETCH.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_INFO.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_PARAM.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_SOAP.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GO_UTILS.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\GODBC32.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=.\driver.DEF
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\SOCK\SOCK_CLI.CPP
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLLEX.cpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLNLST.cpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLPARSE.cpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLPHELP.cpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLTREE.cpp
-# End Source File
-# End Group
-# Begin Group "Header Files"
-
-# PROP Default_Filter "h;hpp;hxx;hm;inl"
-# Begin Source File
-
-SOURCE=..\Common\char.h
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\common.hpp
-# End Source File
-# Begin Source File
-
-SOURCE=.\GODBC.H
-# End Source File
-# Begin Source File
-
-SOURCE=.\resource.h
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\SOCK\SOCK_CLI.HPP
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\SOCK\SOCK_CLI.HXX
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\types.h
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\xmllex.hpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\XMLPARSE.H
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\xmlparse.hpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\xmlparse.hxx
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\xmltree.hpp
-# End Source File
-# Begin Source File
-
-SOURCE=..\Common\XML\xmltree.hxx
-# End Source File
-# End Group
-# Begin Group "Resource Files"
-
-# PROP Default_Filter "ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe"
-# Begin Source File
-
-SOURCE=.\GODBC.RC
-# End Source File
-# End Group
-# End Target
-# End Project
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/driver.vcxproj
----------------------------------------------------------------------
diff --git a/odbc/Driver/driver.vcxproj b/odbc/Driver/driver.vcxproj
index bbc1c3e..0e60c0c 100644
--- a/odbc/Driver/driver.vcxproj
+++ b/odbc/Driver/driver.vcxproj
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/driver.vcxproj.filters
----------------------------------------------------------------------
diff --git a/odbc/Driver/driver.vcxproj.filters b/odbc/Driver/driver.vcxproj.filters
index 5f3debd..a60e5e1 100644
--- a/odbc/Driver/driver.vcxproj.filters
+++ b/odbc/Driver/driver.vcxproj.filters
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/driver.vcxproj.user
----------------------------------------------------------------------
diff --git a/odbc/Driver/driver.vcxproj.user b/odbc/Driver/driver.vcxproj.user
index a375ae3..27d8ac0 100644
--- a/odbc/Driver/driver.vcxproj.user
+++ b/odbc/Driver/driver.vcxproj.user
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Driver/version.props
----------------------------------------------------------------------
diff --git a/odbc/Driver/version.props b/odbc/Driver/version.props
index 1329bf6..ceca789 100644
--- a/odbc/Driver/version.props
+++ b/odbc/Driver/version.props
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<CppRestBaseFileName>cpprest</CppRestBaseFileName>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer(64bit)/Installer(64bit).isl
----------------------------------------------------------------------
diff --git a/odbc/Installer(64bit)/Installer(64bit).isl b/odbc/Installer(64bit)/Installer(64bit).isl
index d8b25ee..e3f0bfc 100644
--- a/odbc/Installer(64bit)/Installer(64bit).isl
+++ b/odbc/Installer(64bit)/Installer(64bit).isl
@@ -1137,6 +1137,7 @@
<col def="I4">ISAttributes</col>
<col def="S255">ISFolderName</col>
<row><td>ALLUSERSPROFILE</td><td>TARGETDIR</td><td>.:ALLUSE~1|All Users</td><td/><td>0</td><td/></row>
+ <row><td>APACHE_KYLIN</td><td>ProgramFiles64Folder</td><td>APACHE~1|Apache Kylin</td><td/><td>0</td><td/></row>
<row><td>AdminToolsFolder</td><td>TARGETDIR</td><td>.:Admint~1|AdminTools</td><td/><td>0</td><td/></row>
<row><td>AppDataFolder</td><td>TARGETDIR</td><td>.:APPLIC~1|Application Data</td><td/><td>0</td><td/></row>
<row><td>CommonAppDataFolder</td><td>TARGETDIR</td><td>.:Common~1|CommonAppData</td><td/><td>0</td><td/></row>
@@ -1149,13 +1150,14 @@
<row><td>FavoritesFolder</td><td>TARGETDIR</td><td>.:FAVORI~1|Favorites</td><td/><td>0</td><td/></row>
<row><td>FontsFolder</td><td>TARGETDIR</td><td>.:Fonts</td><td/><td>0</td><td/></row>
<row><td>GlobalAssemblyCache</td><td>TARGETDIR</td><td>.:Global~1|GlobalAssemblyCache</td><td/><td>0</td><td/></row>
- <row><td>INSTALLDIR</td><td>KYLINODBCDRIVER__X64_</td><td>.</td><td/><td>0</td><td/></row>
+ <row><td>INSTALLDIR</td><td>KYLINODBCDRIVER__X64_1</td><td>.</td><td/><td>0</td><td/></row>
<row><td>ISCommonFilesFolder</td><td>CommonFilesFolder</td><td>Instal~1|InstallShield</td><td/><td>0</td><td/></row>
<row><td>ISMyCompanyDir</td><td>ProgramFilesFolder</td><td>MYCOMP~1|My Company Name</td><td/><td>0</td><td/></row>
<row><td>ISMyProductDir</td><td>ISMyCompanyDir</td><td>MYPROD~1|My Product Name</td><td/><td>0</td><td/></row>
<row><td>ISYourDataBaseDir</td><td>INSTALLDIR</td><td>Database</td><td/><td>0</td><td/></row>
<row><td>KYLINODBCDRIVER</td><td>EBAY1</td><td>KYLINO~1|KylinODBCDriver</td><td/><td>0</td><td/></row>
<row><td>KYLINODBCDRIVER__X64_</td><td>EBAY1</td><td>KYLINO~1|KylinODBCDriver (x64)</td><td/><td>0</td><td/></row>
+ <row><td>KYLINODBCDRIVER__X64_1</td><td>APACHE_KYLIN</td><td>KYLINO~1|KylinODBCDriver (x64)</td><td/><td>0</td><td/></row>
<row><td>LocalAppDataFolder</td><td>TARGETDIR</td><td>.:LocalA~1|LocalAppData</td><td/><td>0</td><td/></row>
<row><td>MY_PRODUCT_NAME</td><td>EBAY</td><td>MYPROD~1|My Product Name</td><td/><td>0</td><td/></row>
<row><td>MyPicturesFolder</td><td>TARGETDIR</td><td>.:MyPict~1|MyPictures</td><td/><td>0</td><td/></row>
@@ -2660,7 +2662,7 @@
<col def="I2">Encoded</col>
<col def="S0">Comment</col>
<col def="I4">TimeStamp</col>
- <row><td>COMPANY_NAME</td><td>1033</td><td>kylinolap</td><td>0</td><td/><td>1965175470</td></row>
+ <row><td>COMPANY_NAME</td><td>1033</td><td>Apache Kylin</td><td>0</td><td/><td>-182245712</td></row>
<row><td>DN_AlwaysInstall</td><td>1033</td><td>Always Install</td><td>0</td><td/><td>-761093519</td></row>
<row><td>IDPROP_EXPRESS_LAUNCH_CONDITION_COLOR</td><td>1033</td><td>The color settings of your system are not adequate for running [ProductName].</td><td>0</td><td/><td>-761093519</td></row>
<row><td>IDPROP_EXPRESS_LAUNCH_CONDITION_OS</td><td>1033</td><td>The operating system is not adequate for running [ProductName].</td><td>0</td><td/><td>-761093519</td></row>
@@ -3760,8 +3762,8 @@
<row><td>IDS__TargetReq_DESC_PROCESSOR</td><td>1033</td><td>The processor is not adequate for running [ProductName].</td><td>0</td><td/><td>-761093519</td></row>
<row><td>IDS__TargetReq_DESC_RAM</td><td>1033</td><td>The amount of RAM is not adequate for running [ProductName].</td><td>0</td><td/><td>-761093519</td></row>
<row><td>IDS__TargetReq_DESC_RESOLUTION</td><td>1033</td><td>The screen resolution is not adequate for running [ProductName].</td><td>0</td><td/><td>-761093519</td></row>
- <row><td>ID_STRING1</td><td>1033</td><td>http://kylin.io</td><td>0</td><td/><td>1965168496</td></row>
- <row><td>ID_STRING2</td><td>1033</td><td>kylinolap</td><td>0</td><td/><td>1965175470</td></row>
+ <row><td>ID_STRING1</td><td>1033</td><td>http://kylin.apache.org/</td><td>0</td><td/><td>-182229328</td></row>
+ <row><td>ID_STRING2</td><td>1033</td><td>Apache Kylin</td><td>0</td><td/><td>-182245712</td></row>
<row><td>IIDS_UITEXT_FeatureUninstalled</td><td>1033</td><td>This feature will remain uninstalled.</td><td>0</td><td/><td>-761093519</td></row>
</table>
@@ -4376,7 +4378,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA=
<row><td>PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS</td><td>##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##</td><td/></row>
<row><td>ProductCode</td><td>{C8B1B296-2F8B-423F-97D5-429C9B6D2AE6}</td><td/></row>
<row><td>ProductName</td><td>KylinODBCDriver (x64)</td><td/></row>
- <row><td>ProductVersion</td><td>0.7.0000</td><td/></row>
+ <row><td>ProductVersion</td><td>1.1.0000</td><td/></row>
<row><td>ProgressType0</td><td>install</td><td/></row>
<row><td>ProgressType1</td><td>Installing</td><td/></row>
<row><td>ProgressType2</td><td>installed</td><td/></row>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer(64bit)/Installer(64bit).isproj
----------------------------------------------------------------------
diff --git a/odbc/Installer(64bit)/Installer(64bit).isproj b/odbc/Installer(64bit)/Installer(64bit).isproj
index 9e4481b..7c4b2f6 100644
--- a/odbc/Installer(64bit)/Installer(64bit).isproj
+++ b/odbc/Installer(64bit)/Installer(64bit).isproj
@@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<!-- Select a Product Configuration -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer(64bit)/Installer(64bit)/Express/Interm/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer(64bit)/Installer(64bit)/Express/Interm/.gitignore b/odbc/Installer(64bit)/Installer(64bit)/Express/Interm/.gitignore
deleted file mode 100644
index d6fd0a4..0000000
--- a/odbc/Installer(64bit)/Installer(64bit)/Express/Interm/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.Exe
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/LogFiles/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/LogFiles/.gitignore b/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/LogFiles/.gitignore
deleted file mode 100644
index 2211df6..0000000
--- a/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/LogFiles/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.txt
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/Reports/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/Reports/.gitignore b/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/Reports/.gitignore
deleted file mode 100644
index 43d1567..0000000
--- a/odbc/Installer(64bit)/Installer(64bit)/Express/SingleImage/Reports/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.htm
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer/Installer.isl
----------------------------------------------------------------------
diff --git a/odbc/Installer/Installer.isl b/odbc/Installer/Installer.isl
index 8beb61d..1a4c4f9 100644
--- a/odbc/Installer/Installer.isl
+++ b/odbc/Installer/Installer.isl
@@ -1129,6 +1129,7 @@
<col def="I4">ISAttributes</col>
<col def="S255">ISFolderName</col>
<row><td>ALLUSERSPROFILE</td><td>TARGETDIR</td><td>.:ALLUSE~1|All Users</td><td/><td>0</td><td/></row>
+ <row><td>APACHE_KYLIN</td><td>ProgramFilesFolder</td><td>APACHE~1|Apache Kylin</td><td/><td>0</td><td/></row>
<row><td>AdminToolsFolder</td><td>TARGETDIR</td><td>.:Admint~1|AdminTools</td><td/><td>0</td><td/></row>
<row><td>AppDataFolder</td><td>TARGETDIR</td><td>.:APPLIC~1|Application Data</td><td/><td>0</td><td/></row>
<row><td>CommonAppDataFolder</td><td>TARGETDIR</td><td>.:Common~1|CommonAppData</td><td/><td>0</td><td/></row>
@@ -1139,11 +1140,12 @@
<row><td>FavoritesFolder</td><td>TARGETDIR</td><td>.:FAVORI~1|Favorites</td><td/><td>0</td><td/></row>
<row><td>FontsFolder</td><td>TARGETDIR</td><td>.:Fonts</td><td/><td>0</td><td/></row>
<row><td>GlobalAssemblyCache</td><td>TARGETDIR</td><td>.:Global~1|GlobalAssemblyCache</td><td/><td>0</td><td/></row>
- <row><td>INSTALLDIR</td><td>KYLINODBCDRIVER__X86_</td><td>.</td><td/><td>0</td><td/></row>
+ <row><td>INSTALLDIR</td><td>KYLINODBCDRIVER__X86_1</td><td>.</td><td/><td>0</td><td/></row>
<row><td>ISCommonFilesFolder</td><td>CommonFilesFolder</td><td>Instal~1|InstallShield</td><td/><td>0</td><td/></row>
<row><td>ISYourDataBaseDir</td><td>INSTALLDIR</td><td>Database</td><td/><td>0</td><td/></row>
<row><td>KYLINODBCDRIVER</td><td>kylinolap</td><td>KYLINO~1|KylinODBCDriver</td><td/><td>0</td><td/></row>
<row><td>KYLINODBCDRIVER__X86_</td><td>kylinolap</td><td>KYLINO~1|KylinODBCDriver (x86)</td><td/><td>0</td><td/></row>
+ <row><td>KYLINODBCDRIVER__X86_1</td><td>APACHE_KYLIN</td><td>KYLINO~1|KylinODBCDriver (x86)</td><td/><td>0</td><td/></row>
<row><td>LocalAppDataFolder</td><td>TARGETDIR</td><td>.:LocalA~1|LocalAppData</td><td/><td>0</td><td/></row>
<row><td>MY_PRODUCT_NAME</td><td>kylinolap</td><td>MYPROD~1|My Product Name</td><td/><td>0</td><td/></row>
<row><td>MyPicturesFolder</td><td>TARGETDIR</td><td>.:MyPict~1|MyPictures</td><td/><td>0</td><td/></row>
@@ -2641,7 +2643,7 @@
<col def="I2">Encoded</col>
<col def="S0">Comment</col>
<col def="I4">TimeStamp</col>
- <row><td>COMPANY_NAME</td><td>1033</td><td>kylinolap</td><td>0</td><td/><td>1965213424</td></row>
+ <row><td>COMPANY_NAME</td><td>1033</td><td>Apache Kylin</td><td>0</td><td/><td>-182247760</td></row>
<row><td>DN_AlwaysInstall</td><td>1033</td><td>Always Install</td><td>0</td><td/><td>-1977438387</td></row>
<row><td>IDPROP_EXPRESS_LAUNCH_CONDITION_COLOR</td><td>1033</td><td>The color settings of your system are not adequate for running [ProductName].</td><td>0</td><td/><td>-1977438387</td></row>
<row><td>IDPROP_EXPRESS_LAUNCH_CONDITION_OS</td><td>1033</td><td>The operating system is not adequate for running [ProductName].</td><td>0</td><td/><td>-1977438387</td></row>
@@ -3741,8 +3743,8 @@
<row><td>IDS__TargetReq_DESC_PROCESSOR</td><td>1033</td><td>The processor is not adequate for running [ProductName].</td><td>0</td><td/><td>-1977438387</td></row>
<row><td>IDS__TargetReq_DESC_RAM</td><td>1033</td><td>The amount of RAM is not adequate for running [ProductName].</td><td>0</td><td/><td>-1977438387</td></row>
<row><td>IDS__TargetReq_DESC_RESOLUTION</td><td>1033</td><td>The screen resolution is not adequate for running [ProductName].</td><td>0</td><td/><td>-1977438387</td></row>
- <row><td>ID_STRING1</td><td>1033</td><td>http://kylin.io</td><td>0</td><td/><td>1965188880</td></row>
- <row><td>ID_STRING2</td><td>1033</td><td>kylinolap</td><td>0</td><td/><td>1965213424</td></row>
+ <row><td>ID_STRING1</td><td>1033</td><td>http://kylin.apache.org/</td><td>0</td><td/><td>-182233424</td></row>
+ <row><td>ID_STRING2</td><td>1033</td><td>Apache Kylin</td><td>0</td><td/><td>-182247760</td></row>
<row><td>IIDS_UITEXT_FeatureUninstalled</td><td>1033</td><td>This feature will remain uninstalled.</td><td>0</td><td/><td>-1977438387</td></row>
</table>
@@ -4357,7 +4359,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA=
<row><td>PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS</td><td>##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##</td><td/></row>
<row><td>ProductCode</td><td>{A920FB5E-591B-4537-901D-7D0303088884}</td><td/></row>
<row><td>ProductName</td><td>KylinODBCDriver (x86)</td><td/></row>
- <row><td>ProductVersion</td><td>0.7.0000</td><td/></row>
+ <row><td>ProductVersion</td><td>1.1.0000</td><td/></row>
<row><td>ProgressType0</td><td>install</td><td/></row>
<row><td>ProgressType1</td><td>Installing</td><td/></row>
<row><td>ProgressType2</td><td>installed</td><td/></row>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer/Installer.isproj
----------------------------------------------------------------------
diff --git a/odbc/Installer/Installer.isproj b/odbc/Installer/Installer.isproj
index 9e4481b..7c4b2f6 100644
--- a/odbc/Installer/Installer.isproj
+++ b/odbc/Installer/Installer.isproj
@@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<!-- Select a Product Configuration -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer/Installer/Express/Interm/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer/Installer/Express/Interm/.gitignore b/odbc/Installer/Installer/Express/Interm/.gitignore
deleted file mode 100644
index d6fd0a4..0000000
--- a/odbc/Installer/Installer/Express/Interm/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.Exe
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer/Installer/Express/SingleImage/LogFiles/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer/Installer/Express/SingleImage/LogFiles/.gitignore b/odbc/Installer/Installer/Express/SingleImage/LogFiles/.gitignore
deleted file mode 100644
index 2211df6..0000000
--- a/odbc/Installer/Installer/Express/SingleImage/LogFiles/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.txt
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/Installer/Installer/Express/SingleImage/Reports/.gitignore
----------------------------------------------------------------------
diff --git a/odbc/Installer/Installer/Express/SingleImage/Reports/.gitignore b/odbc/Installer/Installer/Express/SingleImage/Reports/.gitignore
deleted file mode 100644
index 43d1567..0000000
--- a/odbc/Installer/Installer/Express/SingleImage/Reports/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.htm
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/KylinODBC.sln
----------------------------------------------------------------------
diff --git a/odbc/KylinODBC.sln b/odbc/KylinODBC.sln
index 35091ac..db7f65d 100644
--- a/odbc/KylinODBC.sln
+++ b/odbc/KylinODBC.sln
@@ -1,5 +1,19 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
# Visual Studio 2012
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Common", "Common\Common.vcxproj", "{6EE1C577-1826-4EA2-86AC-FE8047D396A9}"
EndProject
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/README.md
----------------------------------------------------------------------
diff --git a/odbc/README.md b/odbc/README.md
index 585b061..ccee034 100644
--- a/odbc/README.md
+++ b/odbc/README.md
@@ -1,4 +1,4 @@
-# Abount KylinODBC
+# About KylinODBC
ODBC Driver to interactive with Kylin REST server
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/ColorPrint.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/ColorPrint.cpp b/odbc/TestDLL/ColorPrint.cpp
index 17036b2..ceb245b 100644
--- a/odbc/TestDLL/ColorPrint.cpp
+++ b/odbc/TestDLL/ColorPrint.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include <stdio.h>
#include <windows.h>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/ColorPrint.h
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/ColorPrint.h b/odbc/TestDLL/ColorPrint.h
index 472be69..7ad5710 100644
--- a/odbc/TestDLL/ColorPrint.h
+++ b/odbc/TestDLL/ColorPrint.h
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#pragma once
void setPrintColorRED();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/CompareQueryTests.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/CompareQueryTests.cpp b/odbc/TestDLL/CompareQueryTests.cpp
index dbdab90..c7788a2 100644
--- a/odbc/TestDLL/CompareQueryTests.cpp
+++ b/odbc/TestDLL/CompareQueryTests.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include "Tests.h"
#include <vector>
@@ -545,4 +545,68 @@ void crossValidate() {
fprintf ( stdout, wstring2string ( *iter ).c_str() );
fprintf ( stdout, "\n\n" );
}
+}
+
+void validateSQLGetTypeInfo () {
+ Sleep ( 1000 );
+ SQLRETURN status;
+ SQLHANDLE hEnv = 0;
+ SQLHANDLE hConn = 0;
+ SQLHANDLE hStmt = 0;
+ wchar_t szConnStrOut[1024];
+ SQLSMALLINT x;
+
+ // BEFORE U CONNECT
+ // allocate ENVIRONMENT
+ status = SQLAllocHandle ( SQL_HANDLE_ENV, SQL_NULL_HANDLE, &hEnv );
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_ENV, hEnv, status, "" );
+ // set the ODBC version for behaviour expected
+ status = SQLSetEnvAttr ( hEnv, SQL_ATTR_ODBC_VERSION, ( SQLPOINTER ) SQL_OV_ODBC3, 0 );
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_ENV, hEnv, status, "" );
+ // allocate CONNECTION
+ status = SQLAllocHandle ( SQL_HANDLE_DBC, hEnv, &hConn );
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_ENV, hEnv, status, "" );
+ #ifdef _WIN64
+ // ----------- real connection takes place at this point
+ // ----------- option 1: user is prompted for DSN & options
+ status = SQLDriverConnect ( hConn, GetDesktopWindow(),
+ ( unsigned char* ) "",
+ SQL_NTS, szConnStrOut, 1024, &x,
+ SQL_DRIVER_COMPLETE );
+ #else
+ status = SQLDriverConnectW ( hConn , GetDesktopWindow(),
+ //L"DSN=testDSN;",
+ L"DRIVER={KylinODBCDriver};PROJECT=default;UID=ADMIN;SERVER=http://localhost;PORT=80;",
+ SQL_NTS, szConnStrOut, 1024, &x,
+ SQL_DRIVER_COMPLETE );
+ #endif
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_DBC, hConn, status, "" );
+ // CONGRATUALTIONS ---- u r connected to a DBMS via an ODBC driver
+ // allocate STATEMENT
+ status = SQLAllocHandle ( SQL_HANDLE_STMT, hConn, &hStmt );
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_DBC, hConn, status, "" );
+ // execute the statement
+ //status = SQLExecDirect ( hStmt, ( unsigned char* )argv[1], SQL_NTS );
+ status = SQLGetTypeInfoW ( hStmt, SQL_ALL_TYPES);
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_STMT, hConn, status, "" );
+ // check for error
+ ODBC_CHK_ERROR ( SQL_HANDLE_STMT, hStmt, status, "" );
+Cleanup:
+
+ if ( hStmt )
+ { SQLFreeHandle ( SQL_HANDLE_STMT, hStmt ); }
+
+ if ( hConn )
+ { SQLFreeHandle ( SQL_HANDLE_DBC, hConn ); }
+
+ if ( hEnv )
+ { SQLFreeHandle ( SQL_HANDLE_ENV, hEnv ); }
+
+ return;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/Report.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/Report.cpp b/odbc/TestDLL/Report.cpp
index a637653..9df18f2 100644
--- a/odbc/TestDLL/Report.cpp
+++ b/odbc/TestDLL/Report.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include "Tests.h"
void report ( const char* msg ) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/RestAPITest.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/RestAPITest.cpp b/odbc/TestDLL/RestAPITest.cpp
index 31a004f..c720acf 100644
--- a/odbc/TestDLL/RestAPITest.cpp
+++ b/odbc/TestDLL/RestAPITest.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include "Tests.h"
void restAPITest() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/SimpleQueryTest.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/SimpleQueryTest.cpp b/odbc/TestDLL/SimpleQueryTest.cpp
index 4ec2b23..3df7950 100644
--- a/odbc/TestDLL/SimpleQueryTest.cpp
+++ b/odbc/TestDLL/SimpleQueryTest.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include "Tests.h"
void simpleQueryTest() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/Source.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/Source.cpp b/odbc/TestDLL/Source.cpp
index 389ebd5..e2ce6c3 100644
--- a/odbc/TestDLL/Source.cpp
+++ b/odbc/TestDLL/Source.cpp
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#include "vld.h"
#include "Tests.h"
using namespace std;
@@ -27,5 +27,6 @@ int main() {
//regression test:
//simpleQueryTest();
//queryFlowTest();
- crossValidate();
+ //crossValidate();
+ validateSQLGetTypeInfo();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/TestDLL.vcxproj
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/TestDLL.vcxproj b/odbc/TestDLL/TestDLL.vcxproj
index c1e27c7..659cbc2 100644
--- a/odbc/TestDLL/TestDLL.vcxproj
+++ b/odbc/TestDLL/TestDLL.vcxproj
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
@@ -103,6 +122,7 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>$(SolutionDir)\Common\$(Platform)\$(Configuration)\Common.lib;$(SolutionDir)\Driver\$(Platform)\$(Configuration)\driver.lib;$(CPPREST_HOME)\Binaries\Win32\Release\cpprest110_2_0.lib;winhttp.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib</AdditionalDependencies>
+ <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/TestDLL.vcxproj.filters
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/TestDLL.vcxproj.filters b/odbc/TestDLL/TestDLL.vcxproj.filters
index ee88b0a..d8a7dc6 100644
--- a/odbc/TestDLL/TestDLL.vcxproj.filters
+++ b/odbc/TestDLL/TestDLL.vcxproj.filters
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/TestDLL.vcxproj.user
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/TestDLL.vcxproj.user b/odbc/TestDLL/TestDLL.vcxproj.user
index 1b93756..38833e1 100644
--- a/odbc/TestDLL/TestDLL.vcxproj.user
+++ b/odbc/TestDLL/TestDLL.vcxproj.user
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LocalDebuggerEnvironment>PATH=$(SolutionDir)\Driver\$(Platform)\$(Configuration)\;%PATH%</LocalDebuggerEnvironment>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/Tests.h
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/Tests.h b/odbc/TestDLL/Tests.h
index 98fecee..5b8bbe4 100644
--- a/odbc/TestDLL/Tests.h
+++ b/odbc/TestDLL/Tests.h
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
#pragma once
#define prod_KServerAddr ""
@@ -45,3 +45,4 @@ void simpleQueryTest();
void queryFlowTest();
void restAPITest();
void crossValidate();
+void validateSQLGetTypeInfo();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/TestDLL/testqueries.txt
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/testqueries.txt b/odbc/TestDLL/testqueries.txt
index 4b71e89..ca3ca32 100644
--- a/odbc/TestDLL/testqueries.txt
+++ b/odbc/TestDLL/testqueries.txt
@@ -1,13 +1,25 @@
-
-
-
-
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
select lstg_format_name, sum(price) as GMV from test_kylin_fact where lstg_format_name='FP-GTC' group by lstg_format_name order by lstg_format_name
select LSTG_FORMAT_NAME, sum(price)as GMV, count(1) as TRANS_CNT from test_kylin_fact group by LSTG_FORMAT_NAME order by LSTG_FORMAT_NAME
-
select test_kylin_fact.lstg_format_name, sum(price) as GMV, count(*) as TRANS_CNT from test_kylin_fact group by test_kylin_fact.lstg_format_name order by test_kylin_fact.lstg_format_name
select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV , count(*) as TRANS_CNT from test_kylin_fact group by test_kylin_fact.lstg_format_name having sum(price)>5000 order by test_kylin_fact.lstg_format_name
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/doc/reference/ODBC Driver Development.docx
----------------------------------------------------------------------
diff --git a/odbc/doc/reference/ODBC Driver Development.docx b/odbc/doc/reference/ODBC Driver Development.docx
deleted file mode 100644
index a88c953..0000000
Binary files a/odbc/doc/reference/ODBC Driver Development.docx and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e161aac/odbc/doc/reference/how to customize a odbc driver based on REST service (chinese).docx
----------------------------------------------------------------------
diff --git a/odbc/doc/reference/how to customize a odbc driver based on REST service (chinese).docx b/odbc/doc/reference/how to customize a odbc driver based on REST service (chinese).docx
deleted file mode 100644
index 402313d..0000000
Binary files a/odbc/doc/reference/how to customize a odbc driver based on REST service (chinese).docx and /dev/null differ
[25/31] incubator-kylin git commit: KYLIN-999 Add apache license
header to hadoop conf files
Posted by sh...@apache.org.
KYLIN-999 Add apache license header to hadoop conf files
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e6246e1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e6246e1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e6246e1a
Branch: refs/heads/KYLIN-1112
Commit: e6246e1a8a6cfd8ffd80de471b8dae8ee39d8b10
Parents: 1edbf47
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Nov 4 14:57:52 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/kylin_hive_conf.xml | 16 ++++++++++++++++
.../test_case_data/localmeta/kylin_job_conf.xml | 16 ++++++++++++++++
.../minicluster/kylin_job_conf.lzo_disabled.xml | 16 ++++++++++++++++
.../minicluster/kylin_job_conf.lzo_enabled.xml | 16 ++++++++++++++++
.../test_case_data/minicluster/kylin_job_conf.xml | 16 ++++++++++++++++
.../test_case_data/sandbox/capacity-scheduler.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/core-site.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/hadoop-policy.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/hbase-policy.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/hbase-site.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/hdfs-site.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/hive-site.xml | 17 ++++++++++++++++-
.../test_case_data/sandbox/kylin_hive_conf.xml | 16 ++++++++++++++++
examples/test_case_data/sandbox/kylin_job_conf.xml | 16 ++++++++++++++++
examples/test_case_data/sandbox/mapred-site.xml | 17 ++++++++++++++++-
examples/test_case_data/sandbox/yarn-site.xml | 17 ++++++++++++++++-
16 files changed, 256 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/localmeta/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_hive_conf.xml b/examples/test_case_data/localmeta/kylin_hive_conf.xml
index 3cb5242..61ef949 100644
--- a/examples/test_case_data/localmeta/kylin_hive_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_hive_conf.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
<name>dfs.replication</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/localmeta/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_job_conf.xml b/examples/test_case_data/localmeta/kylin_job_conf.xml
index 10dfdb7..2f0628b 100644
--- a/examples/test_case_data/localmeta/kylin_job_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_job_conf.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
index f62ce55..c189063 100644
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
+++ b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
index f62ce55..c189063 100644
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
+++ b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/minicluster/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.xml b/examples/test_case_data/minicluster/kylin_job_conf.xml
index f62ce55..c189063 100644
--- a/examples/test_case_data/minicluster/kylin_job_conf.xml
+++ b/examples/test_case_data/minicluster/kylin_job_conf.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index dcd63c1..7cb985c 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 08:09:06 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 0a80866..9aa588c 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 08:08:58 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml
index 4556120..a590979 100644
--- a/examples/test_case_data/sandbox/hadoop-policy.xml
+++ b/examples/test_case_data/sandbox/hadoop-policy.xml
@@ -1,4 +1,19 @@
-<!--Fri Apr 24 12:58:40 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml
index ae974bb..178b559 100644
--- a/examples/test_case_data/sandbox/hbase-policy.xml
+++ b/examples/test_case_data/sandbox/hbase-policy.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 06:14:03 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 8d763f9..46d5345 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 06:14:00 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index db4da75..1175fff 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 06:14:02 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index 21fc81a..f4c7738 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -1,4 +1,19 @@
-<!--Fri Apr 24 13:44:42 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index faa00d9..f7ae507 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
<name>dfs.replication</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index c851565..bd947af 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index cba5aaa..31d2a39 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 08:09:05 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e6246e1a/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index a8cda86..8256158 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -1,4 +1,19 @@
-<!--Wed May 27 08:09:01 2015-->
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<property>
[23/31] incubator-kylin git commit: KYLIN-752 Fix test case
Posted by sh...@apache.org.
KYLIN-752 Fix test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/42f12543
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/42f12543
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/42f12543
Branch: refs/heads/KYLIN-1112
Commit: 42f125433fc20083f886080615d35405e341e8c0
Parents: 07038d9
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 2 17:40:28 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../storage/translate/ColumnValueRangeTest.java | 112 +++++++++++++++++++
1 file changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f12543/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
new file mode 100644
index 0000000..aaf07fe
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
@@ -0,0 +1,112 @@
+package org.apache.kylin.storage.translate;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.Test;
+
+public class ColumnValueRangeTest {
+
+ @Test
+ public void testPreEvaluateWithDict() {
+ TblColRef col = mockupTblColRef();
+ Dictionary<String> dict = mockupDictionary(col, "CN", "US");
+
+ ColumnValueRange r1 = new ColumnValueRange(col, set("CN", "US", "Other"), FilterOperatorEnum.EQ);
+ r1.preEvaluateWithDict(dict);
+ assertEquals(set("CN", "US"), r1.getEqualValues());
+
+ // less than rounding
+ {
+ ColumnValueRange r2 = new ColumnValueRange(col, set("CN"), FilterOperatorEnum.LT);
+ r2.preEvaluateWithDict(dict);
+ assertEquals(null, r2.getBeginValue());
+ assertEquals("CN", r2.getEndValue());
+
+ ColumnValueRange r3 = new ColumnValueRange(col, set("Other"), FilterOperatorEnum.LT);
+ r3.preEvaluateWithDict(dict);
+ assertEquals(null, r3.getBeginValue());
+ assertEquals("CN", r3.getEndValue());
+
+ ColumnValueRange r4 = new ColumnValueRange(col, set("UT"), FilterOperatorEnum.LT);
+ r4.preEvaluateWithDict(dict);
+ assertEquals(null, r4.getBeginValue());
+ assertEquals("US", r4.getEndValue());
+ }
+
+ // greater than rounding
+ {
+ ColumnValueRange r2 = new ColumnValueRange(col, set("CN"), FilterOperatorEnum.GTE);
+ r2.preEvaluateWithDict(dict);
+ assertEquals("CN", r2.getBeginValue());
+ assertEquals(null, r2.getEndValue());
+
+ ColumnValueRange r3 = new ColumnValueRange(col, set("Other"), FilterOperatorEnum.GTE);
+ r3.preEvaluateWithDict(dict);
+ assertEquals("US", r3.getBeginValue());
+ assertEquals(null, r3.getEndValue());
+
+ ColumnValueRange r4 = new ColumnValueRange(col, set("CI"), FilterOperatorEnum.GTE);
+ r4.preEvaluateWithDict(dict);
+ assertEquals("CN", r4.getBeginValue());
+ assertEquals(null, r4.getEndValue());
+ }
+
+ // ever false check
+ {
+ ColumnValueRange r2 = new ColumnValueRange(col, set("UT"), FilterOperatorEnum.GTE);
+ r2.preEvaluateWithDict(dict);
+ assertTrue(r2.satisfyNone());
+
+ ColumnValueRange r3 = new ColumnValueRange(col, set("CM"), FilterOperatorEnum.LT);
+ r3.preEvaluateWithDict(dict);
+ assertTrue(r3.satisfyNone());
+ }
+ }
+
+ public static Dictionary<String> mockupDictionary(TblColRef col, String... values) {
+ TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+ for (String v : values) {
+ builder.addValue(v);
+ }
+ return builder.build(0);
+ }
+
+ private static Set<String> set(String... values) {
+ HashSet<String> list = new HashSet<String>();
+ list.addAll(Arrays.asList(values));
+ return list;
+ }
+
+ public static TblColRef mockupTblColRef() {
+ TableDesc t = mockupTableDesc("table_a");
+ ColumnDesc c = mockupColumnDesc(t, 1, "col_1", "string");
+ return new TblColRef(c);
+ }
+
+ private static TableDesc mockupTableDesc(String tableName) {
+ TableDesc mockup = new TableDesc();
+ mockup.setName(tableName);
+ return mockup;
+ }
+
+ private static ColumnDesc mockupColumnDesc(TableDesc table, int oneBasedColumnIndex, String name, String datatype) {
+ ColumnDesc desc = new ColumnDesc();
+ String id = "" + oneBasedColumnIndex;
+ desc.setId(id);
+ desc.setName(name);
+ desc.setDatatype(datatype);
+ desc.init(table);
+ return desc;
+ }
+}
[29/31] incubator-kylin git commit: KYLIN-1119 fix
find_hive_dependency.sh
Posted by sh...@apache.org.
KYLIN-1119 fix find_hive_dependency.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/584f24e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/584f24e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/584f24e5
Branch: refs/heads/KYLIN-1112
Commit: 584f24e5e510d7e9e84db31f736dab7276fe7b94
Parents: b0b43cf
Author: honma <ho...@ebay.com>
Authored: Tue Nov 3 15:37:02 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 17 ++++++++++++++---
.../kylin/engine/mr/common/AbstractHadoopJob.java | 2 +-
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/584f24e5/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 98d0860..4982c93 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -19,10 +19,19 @@ do
hive_conf_path=$data
fi
done
-hive_home=`echo $hive_exec_path | awk -F '/lib/' '{print $1}'`/
-hcatalog=`find $hive_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+# in some versions of hdp hcatalog is not in hive's classpath, find it separately
+if [ -z "$HCAT_HOME" ]
+then
+ echo "HCAT_HOME not found, try to find hcatalog path from hdp home"
+ hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+ hcatalog_home=${hdp_home}/hive-hcatalog
+else
+ echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
+ hcatalog_home=${HCAT_HOME}
+fi
+
+hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
if [ -z "$hcatalog" ]
then
@@ -30,6 +39,8 @@ then
exit 1
fi
+
+hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
echo "hive dependency: $hive_dependency"
export hive_dependency
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/584f24e5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index c460d62..8782bbe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -160,7 +160,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
Configuration jobConf = job.getConfiguration();
String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
[20/31] incubator-kylin git commit: fix warnings in pom.xml,
upgrade spring.boot to 1.2.7.RELEASE
Posted by sh...@apache.org.
fix warnings in pom.xml, upgrade spring.boot to 1.2.7.RELEASE
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/33b7b852
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/33b7b852
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/33b7b852
Branch: refs/heads/KYLIN-1112
Commit: 33b7b8521d52f0574091e3295832e4109b2758de
Parents: 5949fe7
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 2 14:21:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
core-common/pom.xml | 4 ----
pom.xml | 10 +---------
server/pom.xml | 3 ++-
3 files changed, 3 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/33b7b852/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index ebcb676..f7444be 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -49,10 +49,6 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </dependency>
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/33b7b852/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7a2313c..8332ee0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,7 +64,6 @@
<commons-cli.version>1.2</commons-cli.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.1</commons-lang3.version>
- <commons-math3.version>3.3</commons-math3.version>
<commons-io.version>2.4</commons-io.version>
<commons-configuration.version>1.9</commons-configuration.version>
<commons-daemon.version>1.0.15</commons-daemon.version>
@@ -72,9 +71,6 @@
<commons-collections4.version>4.0</commons-collections4.version>
<commons-email.version>1.1</commons-email.version>
<commons-math3.version>3.5</commons-math3.version>
- <!--
- <commons-math3.version>3.6-SNAPSHOT</commons-math3.version>
- -->
<!-- Spark -->
<spark.version>1.3.0</spark.version>
@@ -96,6 +92,7 @@
<!-- REST Service -->
<spring.framework.version>3.1.2.RELEASE</spring.framework.version>
+ <spring.boot.version>1.2.7.RELEASE</spring.boot.version>
<!-- Calcite Version -->
<calcite.version>1.4.0-incubating</calcite.version>
@@ -398,11 +395,6 @@
<version>${commons-email.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>${commons-math3.version}</version>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/33b7b852/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 1a4e801..96d874a 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -101,7 +101,7 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
- <version>0.5.0.M6</version>
+ <version>${spring.boot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
@@ -434,6 +434,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring.boot.version}</version>
</plugin>
</plugins>
</build>
[10/31] incubator-kylin git commit: KYLIN-942 enable profiling
backdoor toggles on v2 storage
Posted by sh...@apache.org.
KYLIN-942 enable profiling backdoor toggles on v2 storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6b831ff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6b831ff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6b831ff3
Branch: refs/heads/KYLIN-1112
Commit: 6b831ff3a056f1a41988f0b02eafcc94a1f48fe7
Parents: 7ced7f6
Author: honma <ho...@ebay.com>
Authored: Mon Nov 2 11:40:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 15 +-
.../common/util/MemoryBudgetController.java | 1 +
.../cube/inmemcubing/InMemCubeBuilder.java | 5 +-
.../apache/kylin/gridtable/EmptyGTScanner.java | 58 +++
.../kylin/gridtable/GTAggregateScanner.java | 47 +--
.../apache/kylin/gridtable/GTScanRequest.java | 49 ++-
.../common/coprocessor/CoprocessorBehavior.java | 10 +
.../observer/AggregateRegionObserver.java | 9 +-
.../observer/AggregationScanner.java | 42 +-
.../coprocessor/observer/ObserverBehavior.java | 10 -
.../coprocessor/observer/ObserverEnabler.java | 7 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 23 +-
.../coprocessor/endpoint/CubeVisitService.java | 11 +-
.../endpoint/generated/CubeVisitProtos.java | 386 +++++++++++++------
.../endpoint/protobuf/CubeVisit.proto | 8 +-
.../observer/AggregateRegionObserverTest.java | 5 +-
16 files changed, 489 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index bac7258..1e26557 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,18 +20,23 @@ package org.apache.kylin.common.debug;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
*/
public class BackdoorToggles {
+ private static final Logger logger = LoggerFactory.getLogger(BackdoorToggles.class);
+
private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
public static void setToggles(Map<String, String> toggles) {
_backdoorToggles.set(toggles);
}
- public static String getObserverBehavior() {
- return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ public static String getCoprocessorBehavior() {
+ return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
}
public static String getHbaseCubeQueryVersion() {
@@ -101,14 +106,14 @@ public class BackdoorToggles {
public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL";
/**
- * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ * set DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR/SCAN_FILTER_AGGR_CHECKMEM to control observer behavior for debug/profile usage
*
example:(put it into request body)
"backdoorToggles": {
- "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ "DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR": "SCAN"
}
*/
- public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+ public final static String DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR = "DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR";
/**
* set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index db5996d..2c26666 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,6 +57,7 @@ public class MemoryBudgetController {
public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
public static final int ONE_MB = 1024 * 1024;
+ public static final int ONE_GB = 1024 * 1024 * 1024;
private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 84abd47..d7aa50a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -342,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
- GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
+ GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false);
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + baseCuboidId);
@@ -463,7 +464,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
totalSum[i] = Math.round(total);
}
-
+
}
if (totalSumForSanityCheck == null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
new file mode 100644
index 0000000..b3507c9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class EmptyGTScanner implements IGTScanner {
+ @Override
+ public GTInfo getInfo() {
+ return null;
+ }
+
+ @Override
+ public int getScannedRowCount() {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public GTRecord next() {
+ return null;
+ }
+
+ @Override
+ public void remove() {
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 193a05c..35c1210 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -9,10 +9,8 @@ import java.util.SortedMap;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.HLLCAggregator;
-import org.apache.kylin.metadata.measure.LDCAggregator;
+import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.TopNAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -23,6 +21,7 @@ public class GTAggregateScanner implements IGTScanner {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(GTAggregateScanner.class);
+ private int aggregatedRowCount = 0;
final GTInfo info;
final ImmutableBitSet dimensions; // dimensions to return, can be more than group by
@@ -31,19 +30,20 @@ public class GTAggregateScanner implements IGTScanner {
final String[] metricsAggrFuncs;
final IGTScanner inputScanner;
final AggregationCache aggrCache;
+ final boolean enableMemCheck;
- public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
+ public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, boolean enableMemCheck) {
if (req.hasAggregation() == false)
throw new IllegalStateException();
this.info = inputScanner.getInfo();
- //TODO bug? what if multiple metrics on same column? and what is this?
this.dimensions = req.getColumns().andNot(req.getAggrMetrics());
this.groupBy = req.getAggrGroupBy();
this.metrics = req.getAggrMetrics();
this.metricsAggrFuncs = req.getAggrMetricsFuncs();
this.inputScanner = inputScanner;
this.aggrCache = new AggregationCache();
+ this.enableMemCheck = enableMemCheck;
}
@Override
@@ -71,11 +71,7 @@ public class GTAggregateScanner implements IGTScanner {
/** return the estimate memory size of aggregation cache */
public long getEstimateSizeOfAggrCache() {
- return aggrCache.esitmateMemSize();
- }
-
- public Object[] getTotalSumForSanityCheck() {
- return aggrCache.calculateTotalSumSanityCheck();
+ return aggrCache.estimatedMemSize();
}
class AggregationCache {
@@ -145,6 +141,12 @@ public class GTAggregateScanner implements IGTScanner {
}
void aggregate(GTRecord r) {
+ if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) {
+ if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
+ throw new RuntimeException("AggregationCache exceed 1GB");
+ }
+ }
+
final byte[] key = createKey(r);
MeasureAggregator[] aggrs = aggBufMap.get(key);
if (aggrs == null) {
@@ -162,30 +164,7 @@ public class GTAggregateScanner implements IGTScanner {
return info.codeSystem.newMetricsAggregators(metrics, metricsAggrFuncs);
}
- public Object[] calculateTotalSumSanityCheck() {
- MeasureAggregator[] totalSum = newAggregators();
-
- // skip expensive aggregation
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator || totalSum[i] instanceof TopNAggregator)
- totalSum[i] = null;
- }
-
- for (MeasureAggregator[] entry : aggBufMap.values()) {
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] != null)
- totalSum[i].aggregate(entry[i].getState());
- }
- }
- Object[] result = new Object[totalSum.length];
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] != null)
- result[i] = totalSum[i].getState();
- }
- return result;
- }
-
- public long esitmateMemSize() {
+ public long estimatedMemSize() {
if (aggBufMap.isEmpty())
return 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c81dd63..ba997ec 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
@@ -117,14 +118,52 @@ public class GTScanRequest {
}
public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
+ return decorateScanner(scanner, true, true, true);
+ }
+
+ /**
+ * doFilter,doAggr,doMemCheck are only for profiling use.
+ * in normal cases they are all true.
+ *
+ * Refer to CoprocessorBehavior for explanation
+ */
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean doFilter, boolean doAggr, boolean doMemCheck) throws IOException {
IGTScanner result = scanner;
- if (this.hasFilterPushDown()) {
- result = new GTFilterScanner(result, this);
+ if (!doFilter) { //Skip reading this section if you're not profiling!
+ lookAndForget(result);
+ return new EmptyGTScanner();
+ } else {
+
+ if (this.hasFilterPushDown()) {
+ result = new GTFilterScanner(result, this);
+ }
+
+ if (!doAggr) {//Skip reading this section if you're not profiling!
+ lookAndForget(result);
+ return new EmptyGTScanner();
+ }
+
+ if (this.allowPreAggregation && this.hasAggregation()) {
+ result = new GTAggregateScanner(result, this, doMemCheck);
+ }
+ return result;
}
- if (this.allowPreAggregation && this.hasAggregation()) {
- result = new GTAggregateScanner(result, this);
+ }
+
+ //touch every byte of the cell so that the cost of scanning will be trully reflected
+ private void lookAndForget(IGTScanner scanner) {
+ byte meaninglessByte = 0;
+ for (GTRecord gtRecord : scanner) {
+ for (ByteArray col : gtRecord.getInternal()) {
+ if (col != null) {
+ int endIndex = col.offset() + col.length();
+ for (int i = col.offset(); i < endIndex; ++i) {
+ meaninglessByte += col.array()[i];
+ }
+ }
+ }
}
- return result;
+ System.out.println("meaningless byte is now " + meaninglessByte);
}
public boolean hasFilterPushDown() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
new file mode 100644
index 0000000..df81177
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
@@ -0,0 +1,10 @@
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+/**
+ */
+public enum CoprocessorBehavior {
+ SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
+ SCAN_FILTER_AGGR, //aggregate the result. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 824a7d3..c7b650a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -84,15 +85,15 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
- ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ CoprocessorBehavior coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
try {
byte[] behavior = scan.getAttribute(BEHAVIOR);
if (behavior != null && behavior.length != 0) {
- observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ coprocessorBehavior = CoprocessorBehavior.valueOf(new String(behavior));
}
} catch (Exception e) {
LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
- observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
}
// start/end region operation & sync on scanner is suggested by the
@@ -102,7 +103,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner, observerBehavior);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner, coprocessorBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index bc74ac1..fc1d967 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -38,9 +39,9 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
- private ObserverBehavior behavior;
+ private CoprocessorBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, ObserverBehavior behavior) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, CoprocessorBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
@@ -64,6 +65,8 @@ public class AggregationScanner implements RegionScanner {
ObserverTuple tuple = new ObserverTuple(type);
boolean hasMore = true;
List<Cell> results = new ArrayList<Cell>();
+ byte meaninglessByte = 0;
+
while (hasMore) {
results.clear();
hasMore = innerScanner.nextRaw(results);
@@ -76,21 +79,34 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER.ordinal()) {
- if (filter != null && filter.evaluate(tuple) == false)
- continue;
-
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR.ordinal()) {
- AggrKey aggKey = projector.getAggrKey(results);
- MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
- aggregators.aggregate(bufs, results);
-
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
- aggCache.checkMemoryUsage();
+ if (behavior == CoprocessorBehavior.SCAN) {
+ //touch every byte of the cell so that the cost of scanning will be trully reflected
+ int endIndex = cell.getRowOffset() + cell.getRowLength();
+ for (int i = cell.getRowOffset(); i < endIndex; ++i) {
+ meaninglessByte += cell.getRowArray()[i];
+ }
+ } else {
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal()) {
+ if (filter != null && filter.evaluate(tuple) == false)
+ continue;
+
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ AggrKey aggKey = projector.getAggrKey(results);
+ MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
+ aggregators.aggregate(bufs, results);
+
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ aggCache.checkMemoryUsage();
+ }
}
}
}
}
+
+ if (behavior == CoprocessorBehavior.SCAN) {
+ System.out.println("meaningless byte is now " + meaninglessByte);
+ }
+
return aggCache;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
deleted file mode 100644
index 7630d57..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-/**
- */
-public enum ObserverBehavior {
- SCAN, //only scan data, used for profiling tuple scan speed
- SCAN_FILTER, //only scan+filter used,used for profiling filter speed
- SCAN_FILTER_AGGR, //aggregate the result
- SCAN_FILTER_AGGR_CHECKMEM, //default full operations
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index 7db2416..4750ea4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -35,6 +35,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
@@ -74,14 +75,14 @@ public class ObserverEnabler {
if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
return new ResultScannerAdapter(aggrScanner);
} else {
// debug/profiling purpose
- String toggle = BackdoorToggles.getObserverBehavior();
+ String toggle = BackdoorToggles.getCoprocessorBehavior();
if (toggle == null) {
- toggle = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5674986..77a5db5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.DataFormatException;
import javax.annotation.Nullable;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -47,6 +49,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
@@ -64,11 +67,13 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private GTInfo info;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
+ private int totalScannedCount;
- public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns) {
+ public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, int totalScannedCount) {
this.info = info;
this.blocks = blocks;
this.columns = columns;
+ this.totalScannedCount = totalScannedCount;
}
@Override
@@ -78,7 +83,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@Override
public int getScannedRowCount() {
- return 0;
+ return totalScannedCount;
}
@Override
@@ -155,6 +160,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
}
+ final AtomicInteger totalScannedCount = new AtomicInteger(0);
+
for (int i = 0; i < rawScans.size(); ++i) {
final int shardIndex = i;
final RawScan rawScan = rawScans.get(i);
@@ -169,6 +176,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
+ // debug/profiling purpose
+ String toggle = BackdoorToggles.getCoprocessorBehavior();
+ if (toggle == null) {
+ toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ } else {
+ logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
+ }
+ builder.setBehavior(toggle);
+
Collection<CubeVisitProtos.CubeVisitResponse> results;
try {
results = getResults(builder.build(), hbaseTable, rawScan.startKey, rawScan.endKey);
@@ -182,6 +198,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
for (CubeVisitProtos.CubeVisitResponse result : results) {
+ totalScannedCount.addAndGet(result.getStats().getScannedRowCount());
logger.info(getStatsString(result, shardIndex));
}
@@ -209,7 +226,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
throw new RuntimeException("Visiting cube by endpoint gets interrupted");
}
- return new EndpointResultsAsGTScanner(fullGTInfo, rowBlocks.iterator(), scanRequest.getColumns());
+ return new EndpointResultsAsGTScanner(fullGTInfo, rowBlocks.iterator(), scanRequest.getColumns(), totalScannedCount.get());
}
private String getStatsString(CubeVisitProtos.CubeVisitResponse result, int shardIndex) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index fb497b9..f474139 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -41,6 +41,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.CellListIterator;
import org.apache.kylin.storage.hbase.cube.v2.CubeHBaseRPC;
import org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore;
@@ -56,7 +57,8 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
-@SuppressWarnings("unused")//used in hbase endpoint
+@SuppressWarnings("unused")
+//used in hbase endpoint
public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);
@@ -144,7 +146,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns, hbaseColumnsToGT);
IGTScanner rawScanner = store.scan(scanReq);
- IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);
+
+ CoprocessorBehavior behavior = CoprocessorBehavior.valueOf(request.getBehavior());
+ IGTScanner finalScanner = scanReq.decorateScanner(rawScanner,//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal(),//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal(),//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal());//
ByteBuffer buffer = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 225703d..0923484 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,47 +11,62 @@ public final class CubeVisitProtos {
public interface CubeVisitRequestOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // required bytes gtScanRequest = 1;
+ // required string behavior = 1;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required string behavior = 1;</code>
+ */
+ boolean hasBehavior();
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ java.lang.String getBehavior();
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getBehaviorBytes();
+
+ // required bytes gtScanRequest = 2;
+ /**
+ * <code>required bytes gtScanRequest = 2;</code>
*/
boolean hasGtScanRequest();
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
com.google.protobuf.ByteString getGtScanRequest();
- // required bytes hbaseRawScan = 2;
+ // required bytes hbaseRawScan = 3;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
boolean hasHbaseRawScan();
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
com.google.protobuf.ByteString getHbaseRawScan();
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>
getHbaseColumnsToGTList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index);
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
int getHbaseColumnsToGTCount();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index);
@@ -109,18 +124,23 @@ public final class CubeVisitProtos {
}
case 10: {
bitField0_ |= 0x00000001;
- gtScanRequest_ = input.readBytes();
+ behavior_ = input.readBytes();
break;
}
case 18: {
bitField0_ |= 0x00000002;
- hbaseRawScan_ = input.readBytes();
+ gtScanRequest_ = input.readBytes();
break;
}
case 26: {
- if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ bitField0_ |= 0x00000004;
+ hbaseRawScan_ = input.readBytes();
+ break;
+ }
+ case 34: {
+ if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>();
- mutable_bitField0_ |= 0x00000004;
+ mutable_bitField0_ |= 0x00000008;
}
hbaseColumnsToGT_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.PARSER, extensionRegistry));
break;
@@ -133,7 +153,7 @@ public final class CubeVisitProtos {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
- if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
}
this.unknownFields = unknownFields.build();
@@ -672,68 +692,111 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required bytes gtScanRequest = 1;
- public static final int GTSCANREQUEST_FIELD_NUMBER = 1;
+ // required string behavior = 1;
+ public static final int BEHAVIOR_FIELD_NUMBER = 1;
+ private java.lang.Object behavior_;
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public boolean hasBehavior() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public java.lang.String getBehavior() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ behavior_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getBehaviorBytes() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ behavior_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required bytes gtScanRequest = 2;
+ public static final int GTSCANREQUEST_FIELD_NUMBER = 2;
private com.google.protobuf.ByteString gtScanRequest_;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
- // required bytes hbaseRawScan = 2;
- public static final int HBASERAWSCAN_FIELD_NUMBER = 2;
+ // required bytes hbaseRawScan = 3;
+ public static final int HBASERAWSCAN_FIELD_NUMBER = 3;
private com.google.protobuf.ByteString hbaseRawScan_;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
- public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
+ public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
return hbaseColumnsToGT_.size();
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
return hbaseColumnsToGT_.get(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -741,6 +804,7 @@ public final class CubeVisitProtos {
}
private void initFields() {
+ behavior_ = "";
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
hbaseColumnsToGT_ = java.util.Collections.emptyList();
@@ -750,6 +814,10 @@ public final class CubeVisitProtos {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
+ if (!hasBehavior()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
if (!hasGtScanRequest()) {
memoizedIsInitialized = 0;
return false;
@@ -766,13 +834,16 @@ public final class CubeVisitProtos {
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, gtScanRequest_);
+ output.writeBytes(1, getBehaviorBytes());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, hbaseRawScan_);
+ output.writeBytes(2, gtScanRequest_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeBytes(3, hbaseRawScan_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
- output.writeMessage(3, hbaseColumnsToGT_.get(i));
+ output.writeMessage(4, hbaseColumnsToGT_.get(i));
}
getUnknownFields().writeTo(output);
}
@@ -785,15 +856,19 @@ public final class CubeVisitProtos {
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, gtScanRequest_);
+ .computeBytesSize(1, getBehaviorBytes());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, hbaseRawScan_);
+ .computeBytesSize(2, gtScanRequest_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, hbaseRawScan_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(3, hbaseColumnsToGT_.get(i));
+ .computeMessageSize(4, hbaseColumnsToGT_.get(i));
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
@@ -818,6 +893,11 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest) obj;
boolean result = true;
+ result = result && (hasBehavior() == other.hasBehavior());
+ if (hasBehavior()) {
+ result = result && getBehavior()
+ .equals(other.getBehavior());
+ }
result = result && (hasGtScanRequest() == other.hasGtScanRequest());
if (hasGtScanRequest()) {
result = result && getGtScanRequest()
@@ -843,6 +923,10 @@ public final class CubeVisitProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasBehavior()) {
+ hash = (37 * hash) + BEHAVIOR_FIELD_NUMBER;
+ hash = (53 * hash) + getBehavior().hashCode();
+ }
if (hasGtScanRequest()) {
hash = (37 * hash) + GTSCANREQUEST_FIELD_NUMBER;
hash = (53 * hash) + getGtScanRequest().hashCode();
@@ -965,13 +1049,15 @@ public final class CubeVisitProtos {
public Builder clear() {
super.clear();
- gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
+ behavior_ = "";
bitField0_ = (bitField0_ & ~0x00000001);
- hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
+ gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000002);
+ hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000004);
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
hbaseColumnsToGTBuilder_.clear();
}
@@ -1006,15 +1092,19 @@ public final class CubeVisitProtos {
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
to_bitField0_ |= 0x00000001;
}
- result.gtScanRequest_ = gtScanRequest_;
+ result.behavior_ = behavior_;
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000002;
}
+ result.gtScanRequest_ = gtScanRequest_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
result.hbaseRawScan_ = hbaseRawScan_;
if (hbaseColumnsToGTBuilder_ == null) {
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
}
result.hbaseColumnsToGT_ = hbaseColumnsToGT_;
} else {
@@ -1036,6 +1126,11 @@ public final class CubeVisitProtos {
public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other) {
if (other == org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.getDefaultInstance()) return this;
+ if (other.hasBehavior()) {
+ bitField0_ |= 0x00000001;
+ behavior_ = other.behavior_;
+ onChanged();
+ }
if (other.hasGtScanRequest()) {
setGtScanRequest(other.getGtScanRequest());
}
@@ -1046,7 +1141,7 @@ public final class CubeVisitProtos {
if (!other.hbaseColumnsToGT_.isEmpty()) {
if (hbaseColumnsToGT_.isEmpty()) {
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
ensureHbaseColumnsToGTIsMutable();
hbaseColumnsToGT_.addAll(other.hbaseColumnsToGT_);
@@ -1059,7 +1154,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_.dispose();
hbaseColumnsToGTBuilder_ = null;
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
hbaseColumnsToGTBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getHbaseColumnsToGTFieldBuilder() : null;
@@ -1073,6 +1168,10 @@ public final class CubeVisitProtos {
}
public final boolean isInitialized() {
+ if (!hasBehavior()) {
+
+ return false;
+ }
if (!hasGtScanRequest()) {
return false;
@@ -1103,85 +1202,159 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required bytes gtScanRequest = 1;
+ // required string behavior = 1;
+ private java.lang.Object behavior_ = "";
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public boolean hasBehavior() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public java.lang.String getBehavior() {
+ java.lang.Object ref = behavior_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ behavior_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getBehaviorBytes() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ behavior_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder setBehavior(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ behavior_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder clearBehavior() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ behavior_ = getDefaultInstance().getBehavior();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder setBehaviorBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ behavior_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required bytes gtScanRequest = 2;
private com.google.protobuf.ByteString gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public Builder setGtScanRequest(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000001;
+ bitField0_ |= 0x00000002;
gtScanRequest_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public Builder clearGtScanRequest() {
- bitField0_ = (bitField0_ & ~0x00000001);
+ bitField0_ = (bitField0_ & ~0x00000002);
gtScanRequest_ = getDefaultInstance().getGtScanRequest();
onChanged();
return this;
}
- // required bytes hbaseRawScan = 2;
+ // required bytes hbaseRawScan = 3;
private com.google.protobuf.ByteString hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public Builder setHbaseRawScan(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000004;
hbaseRawScan_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public Builder clearHbaseRawScan() {
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000004);
hbaseRawScan_ = getDefaultInstance().getHbaseRawScan();
onChanged();
return this;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_ =
java.util.Collections.emptyList();
private void ensureHbaseColumnsToGTIsMutable() {
- if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+ if (!((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>(hbaseColumnsToGT_);
- bitField0_ |= 0x00000004;
+ bitField0_ |= 0x00000008;
}
}
@@ -1189,7 +1362,7 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder> hbaseColumnsToGTBuilder_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1199,7 +1372,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1209,7 +1382,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1219,7 +1392,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1236,7 +1409,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1250,7 +1423,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1266,7 +1439,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1283,7 +1456,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1297,7 +1470,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1311,7 +1484,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addAllHbaseColumnsToGT(
java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> values) {
@@ -1325,12 +1498,12 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder clearHbaseColumnsToGT() {
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
onChanged();
} else {
hbaseColumnsToGTBuilder_.clear();
@@ -1338,7 +1511,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder removeHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1351,14 +1524,14 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder getHbaseColumnsToGTBuilder(
int index) {
return getHbaseColumnsToGTFieldBuilder().getBuilder(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -1368,7 +1541,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
@@ -1379,14 +1552,14 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder() {
return getHbaseColumnsToGTFieldBuilder().addBuilder(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder(
int index) {
@@ -1394,7 +1567,7 @@ public final class CubeVisitProtos {
index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder>
getHbaseColumnsToGTBuilderList() {
@@ -1407,7 +1580,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>(
hbaseColumnsToGT_,
- ((bitField0_ & 0x00000004) == 0x00000004),
+ ((bitField0_ & 0x00000008) == 0x00000008),
getParentForChildren(),
isClean());
hbaseColumnsToGT_ = null;
@@ -1606,10 +1779,6 @@ public final class CubeVisitProtos {
}
/**
* Protobuf type {@code CubeVisitResponse.Stats}
- *
- * <pre>
- *all entries in this struct be optional to conveniently add more entries in the future
- * </pre>
*/
public static final class Stats extends
com.google.protobuf.GeneratedMessage
@@ -1983,10 +2152,6 @@ public final class CubeVisitProtos {
}
/**
* Protobuf type {@code CubeVisitResponse.Stats}
- *
- * <pre>
- *all entries in this struct be optional to conveniently add more entries in the future
- * </pre>
*/
public static final class Builder extends
com.google.protobuf.GeneratedMessage.Builder<Builder>
@@ -3055,20 +3220,21 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\215\001\n\020C" +
- "ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
- "\n\014hbaseRawScan\030\002 \002(\014\0223\n\020hbaseColumnsToGT" +
- "\030\003 \003(\0132\031.CubeVisitRequest.IntList\032\027\n\007Int" +
- "List\022\014\n\004ints\030\001 \003(\005\"\304\001\n\021CubeVisitResponse" +
- "\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132" +
- "\030.CubeVisitResponse.Stats\032n\n\005Stats\022\030\n\020se" +
- "rviceStartTime\030\001 \001(\003\022\026\n\016serviceEndTime\030\002",
- " \001(\003\022\027\n\017scannedRowCount\030\003 \001(\005\022\032\n\022aggrega" +
- "tedRowCount\030\004 \001(\0052F\n\020CubeVisitService\0222\n" +
- "\tvisitCube\022\021.CubeVisitRequest\032\022.CubeVisi" +
- "tResponseB`\nEorg.apache.kylin.storage.hb" +
- "ase.cube.v2.coprocessor.endpoint.generat" +
- "edB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "/endpoint/protobuf/CubeVisit.proto\"\237\001\n\020C" +
+ "ubeVisitRequest\022\020\n\010behavior\030\001 \002(\t\022\025\n\rgtS" +
+ "canRequest\030\002 \002(\014\022\024\n\014hbaseRawScan\030\003 \002(\014\0223" +
+ "\n\020hbaseColumnsToGT\030\004 \003(\0132\031.CubeVisitRequ" +
+ "est.IntList\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\304\001\n" +
+ "\021CubeVisitResponse\022\026\n\016compressedRows\030\001 \002" +
+ "(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitResponse.St" +
+ "ats\032n\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026",
+ "\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCoun" +
+ "t\030\003 \001(\005\022\032\n\022aggregatedRowCount\030\004 \001(\0052F\n\020C" +
+ "ubeVisitService\0222\n\tvisitCube\022\021.CubeVisit" +
+ "Request\032\022.CubeVisitResponseB`\nEorg.apach" +
+ "e.kylin.storage.hbase.cube.v2.coprocesso" +
+ "r.endpoint.generatedB\017CubeVisitProtosH\001\210" +
+ "\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3080,7 +3246,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "HbaseColumnsToGT", });
+ new java.lang.String[] { "Behavior", "GtScanRequest", "HbaseRawScan", "HbaseColumnsToGT", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index a4cd39d..4ac6414 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -12,16 +12,16 @@ option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
message CubeVisitRequest {
- required bytes gtScanRequest = 1;
- required bytes hbaseRawScan = 2;
- repeated IntList hbaseColumnsToGT = 3;
+ required string behavior = 1;
+ required bytes gtScanRequest = 2;
+ required bytes hbaseRawScan = 3;
+ repeated IntList hbaseColumnsToGT = 4;
message IntList {
repeated int32 ints = 1;
}
}
message CubeVisitResponse {
- //all entries in this struct be optional to conveniently add more entries in the future
message Stats {
optional int64 serviceStartTime = 1;
optional int64 serviceEndTime = 2;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b831ff3/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 95a9128..a88e147 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -41,6 +41,7 @@ import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -123,7 +124,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -172,7 +173,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[06/31] incubator-kylin git commit: KYLIN-953 Again - Let
MapReduceExecutable pick up patched hadoop config
Posted by sh...@apache.org.
KYLIN-953 Again - Let MapReduceExecutable pick up patched hadoop config
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5fec41f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5fec41f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5fec41f9
Branch: refs/heads/KYLIN-1112
Commit: 5fec41f9dc78ac7ea3594d14666d143fcf2f619d
Parents: f79d7c5
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 2 15:22:42 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5fec41f9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index ebb3900..46557c8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -109,6 +109,7 @@ public class MapReduceExecutable extends AbstractExecutable {
} else {
final Constructor<? extends AbstractHadoopJob> constructor = ClassUtil.forName(mapReduceJobClass, AbstractHadoopJob.class).getConstructor();
final AbstractHadoopJob hadoopJob = constructor.newInstance();
+ hadoopJob.setConf(HadoopUtil.getCurrentConfiguration());
hadoopJob.setAsync(true); // so the ToolRunner.run() returns right away
logger.info("parameters of the MapReduceExecutable:");
logger.info(params);
[04/31] incubator-kylin git commit: bug fix,
SingleValueCache.put() should put local cache first
Posted by sh...@apache.org.
bug fix, SingleValueCache.put() should put local cache first
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9e64da6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9e64da6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9e64da6
Branch: refs/heads/KYLIN-1112
Commit: b9e64da6434a2c163bcfd5066d411299763e1e6a
Parents: 730739f
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Oct 29 11:50:31 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:15 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/restclient/SingleValueCache.java | 2 ++
.../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9e64da6/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 50b0f84..2a3ff06 100644
--- a/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/core-common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -44,6 +44,8 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void put(K key, V value) {
boolean exists = innerCache.containsKey(key);
+
+ innerCache.put(key, value);
if (!exists) {
cacheUpdater.updateCache(key, value, Broadcaster.EVENT.CREATE, syncType, this);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9e64da6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9b66578..b7e7dc5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -431,7 +431,7 @@ public class MetadataManager {
os.close();
is.close();
- srcTableExdMap.putLocal(tableId, tableExdProperties);
+ srcTableExdMap.put(tableId, tableExdProperties);
}
public String appendDBName(String table) {
[16/31] incubator-kylin git commit: KYLIN-919 check model name
duplicate case-insensitive
Posted by sh...@apache.org.
KYLIN-919 check model name duplicate case-insensitive
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/107d3395
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/107d3395
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/107d3395
Branch: refs/heads/KYLIN-1112
Commit: 107d3395a51097be99aa2a9046decc443b8ecb53
Parents: 88d6485
Author: jiazhong <ji...@ebay.com>
Authored: Thu Oct 29 17:29:06 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelSchema.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/107d3395/webapp/app/js/controllers/modelSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelSchema.js b/webapp/app/js/controllers/modelSchema.js
index ccb0013..f4bae9e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -166,7 +166,7 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
var modelName = $scope.modelsManager.selectedModel.name.toUpperCase();
var models = $scope.modelsManager.modelNameList;
- if (models.indexOf(modelName) != -1) {
+ if (models.indexOf(modelName) != -1 || models.indexOf(modelName.toLowerCase()) !=-1) {
SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 'warning');
return false;
}
[14/31] incubator-kylin git commit: KYLIN-1036 enable disabled font
Posted by sh...@apache.org.
KYLIN-1036 enable disabled font
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f79d7c58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f79d7c58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f79d7c58
Branch: refs/heads/KYLIN-1112
Commit: f79d7c584bab367e3bc64a90e3da31ffcd604ab5
Parents: 7148a17
Author: jiazhong <ji...@ebay.com>
Authored: Mon Nov 2 15:19:33 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
webapp/app/css/AdminLTE.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f79d7c58/webapp/app/css/AdminLTE.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index a5d26d1..46f397a 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -21,7 +21,7 @@ body {
body {
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
- /*font-family: 'Source Sans Pro', 'Helvetica Neue', Helvetica, Arial, sans-serif;*/
+ font-family: 'Source Sans Pro', 'Helvetica Neue', Helvetica, Arial, sans-serif;
font-weight: 400;
overflow-x: hidden;
overflow-y: auto;
[17/31] incubator-kylin git commit: KYLIN-953 Let MapReduceExecutable
pick up patched hadoop config
Posted by sh...@apache.org.
KYLIN-953 Let MapReduceExecutable pick up patched hadoop config
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5949fe71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5949fe71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5949fe71
Branch: refs/heads/KYLIN-1112
Commit: 5949fe71e5bf38bc2eaf7d077c00013743efe2e1
Parents: 933e81f
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 2 12:50:08 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../org/apache/kylin/engine/mr/HadoopUtil.java | 20 ++++++++++++++++----
.../engine/mr/common/MapReduceExecutable.java | 7 +++++--
.../engine/mr/steps/MergeStatisticsStep.java | 2 +-
.../HiveToBaseCuboidMapperPerformanceTest.java | 3 ++-
.../engine/mr/steps/MergeCuboidJobTest.java | 3 ++-
.../kylin/engine/mr/steps/NDCuboidJobTest.java | 3 ++-
.../storage/hbase/util/CubeMigrationCLI.java | 3 ++-
.../storage/hbase/steps/CreateHTableTest.java | 3 ++-
.../hbase/steps/CubeHFileMapper2Test.java | 3 ++-
.../storage/hbase/steps/ITHdfsOpsTest.java | 3 ++-
.../steps/RangeKeyDistributionJobTest.java | 3 ++-
12 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index c0bff8a..db885f9 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -52,7 +52,7 @@ public class DictionaryGenerator {
try {
return KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
} catch (Throwable e) {
- return 2000000; // some test case does not KylinConfig setup properly
+ return 5000000; // some test case does not KylinConfig setup properly
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 862f9a0..9ce2bab 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -39,16 +40,27 @@ public class HadoopUtil {
public static Configuration getCurrentConfiguration() {
if (hadoopConfig.get() == null) {
- Configuration conf = new Configuration();
+ Configuration conf = healSickConfig(new Configuration());
- // why we have this hard code?
- conf.set(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, "8");
-
hadoopConfig.set(conf);
}
return hadoopConfig.get();
}
+ private static Configuration healSickConfig(Configuration conf) {
+ // why we have this hard code?
+ conf.set(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, "8");
+
+ // https://issues.apache.org/jira/browse/KYLIN-953
+ if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
+ conf.set("hadoop.tmp.dir", "/tmp");
+ }
+ if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) {
+ conf.set("hbase.fs.tmp.dir", "/tmp");
+ }
+ return conf;
+ }
+
public static FileSystem getFileSystem(String path) throws IOException {
return FileSystem.get(makeURI(path), getCurrentConfiguration());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 4fb3eed..ebb3900 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.constant.JobStepStatusEnum;
import org.apache.kylin.job.exception.ExecuteException;
@@ -72,7 +73,8 @@ public class MapReduceExecutable extends AbstractExecutable {
return;
}
try {
- Job job = new Cluster(new Configuration()).getJob(JobID.forName(mrJobId));
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
+ Job job = new Cluster(conf).getJob(JobID.forName(mrJobId));
if (job.getJobState() == JobStatus.State.FAILED) {
//remove previous mr job info
super.onExecuteStart(executableContext);
@@ -101,7 +103,8 @@ public class MapReduceExecutable extends AbstractExecutable {
Job job;
final Map<String, String> extra = executableManager.getOutput(getId()).getExtra();
if (extra.containsKey(ExecutableConstants.MR_JOB_ID)) {
- job = new Cluster(new Configuration()).getJob(JobID.forName(extra.get(ExecutableConstants.MR_JOB_ID)));
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
+ job = new Cluster(conf).getJob(JobID.forName(extra.get(ExecutableConstants.MR_JOB_ID)));
logger.info("mr_job_id:" + extra.get(ExecutableConstants.MR_JOB_ID + " resumed"));
} else {
final Constructor<? extends AbstractHadoopJob> constructor = ClassUtil.forName(mapReduceJobClass, AbstractHadoopJob.class).getConstructor();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 67c4416..45282df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -76,7 +76,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
final CubeInstance cube = mgr.getCube(getCubeName());
final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
- Configuration conf = new Configuration();
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
index fad521f..c1f50a8 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapperPerformanceTest.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.Ignore;
import org.junit.Test;
@@ -45,7 +46,7 @@ public class HiveToBaseCuboidMapperPerformanceTest {
@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
- Configuration hconf = new Configuration();
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
index 43038a0..ccaa027 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -37,7 +38,7 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
- conf = new Configuration();
+ conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
index c1b8ee3..2a74720 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidJobTest.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -36,7 +37,7 @@ public class NDCuboidJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
- conf = new Configuration();
+ conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 5708514..aec1d5b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -44,6 +44,7 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IEngineAware;
@@ -118,7 +119,7 @@ public class CubeMigrationCLI {
Configuration conf = HBaseConfiguration.create();
hbaseAdmin = new HBaseAdmin(conf);
- hdfsFS = FileSystem.get(new Configuration());
+ hdfsFS = FileSystem.get(HadoopUtil.getCurrentConfiguration());
operations = new ArrayList<Opt>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
index 091d182..c7f9f73 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.storage.hbase.steps.CreateHTableJob;
import org.junit.After;
import org.junit.Before;
@@ -39,7 +40,7 @@ public class CreateHTableTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
- conf = new Configuration();
+ conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local");
this.createTestMetadata();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index 81fa3a6..270928f 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.measure.MeasureCodec;
import org.junit.After;
import org.junit.Before;
@@ -71,7 +72,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
@Test
public void testBasic() throws Exception {
- Configuration hconf = new Configuration();
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV);
CubeHFileMapper mapper = new CubeHFileMapper();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java
index 5db0697..25ac2c6 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -40,7 +41,7 @@ public class ITHdfsOpsTest extends HBaseMetadataTestCase {
this.createTestMetadata();
- Configuration hconf = new Configuration();
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
fileSystem = FileSystem.get(hconf);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5949fe71/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
index 793834a..05d0b08 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.steps.RangeKeyDistributionJob;
import org.junit.After;
import org.junit.Before;
@@ -41,7 +42,7 @@ public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
- conf = new Configuration();
+ conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapred.job.tracker", "local");
[28/31] incubator-kylin git commit: KYLIN-1112 fix inverted index
query bug
Posted by sh...@apache.org.
KYLIN-1112 fix inverted index query bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b7cb41a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b7cb41a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b7cb41a8
Branch: refs/heads/KYLIN-1112
Commit: b7cb41a8b390b8b391fb038f5ce688dc03887089
Parents: bb3aa4f
Author: shaofengshi <sh...@apache.org>
Authored: Wed Nov 4 20:28:11 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/invertedindex/IITest.java | 8 +++----
.../org/apache/kylin/common/util/BytesUtil.java | 10 +++++++++
.../measure/fixedlen/FixedHLLCodec.java | 6 +++++
.../measure/fixedlen/FixedLenMeasureCodec.java | 4 ++++
.../measure/fixedlen/FixedPointLongCodec.java | 8 +++++++
.../endpoint/EndpointAggregators.java | 23 +++++++++++++-------
.../endpoint/EndpointTupleIterator.java | 7 +++---
.../ii/coprocessor/endpoint/IIEndpoint.java | 13 ++++++++---
8 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index 800b615..200156a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -1,6 +1,7 @@
package org.apache.kylin.job.hadoop.invertedindex;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -15,8 +16,6 @@ import com.google.common.base.Function;
import kafka.message.Message;
import kafka.message.MessageAndOffset;
-import com.google.protobuf.HBaseZeroCopyByteString;
-
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -235,9 +234,10 @@ public class IITest extends LocalFileMetadataTestCase {
System.out.println(response.getRowsList().size());
Set<String> answers = Sets.newHashSet("120.4747", "26.8551");
for (IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) {
- byte[] measuresBytes = HBaseZeroCopyByteString.zeroCopyGetBytes(responseRow.getMeasures());
- List<Object> metrics = aggregators.deserializeMetricValues(measuresBytes, 0);
+ ByteBuffer bf = responseRow.getMeasures().asReadOnlyByteBuffer();
+ List<Object> metrics = aggregators.deserializeMetricValues(bf);
Assert.assertTrue(answers.contains(metrics.get(0)));
+ answers.remove(metrics.get(0));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 0d4dba9..e01ce4f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -57,6 +57,16 @@ public class BytesUtil {
return integer;
}
+
+ public static long readLong(ByteBuffer buffer, int size) {
+ long integer = 0;
+ for (int i = 0; i < size; i++) {
+ integer <<= 8;
+ integer |= (long) buffer.get() & 0xFF;
+ }
+ return integer;
+ }
+
public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
for (int i = offset + size - 1; i >= offset; i--) {
bytes[i] = (byte) num;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
index 85dfe14..a0af6a2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -71,4 +71,10 @@ public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter>
public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
}
+
+ @Override
+ public HyperLogLogPlusCounter read(ByteBuffer buffer) {
+ current.readRegistersArray(buffer);
+ return current;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
index ad8c483..54c4eb8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
@@ -20,6 +20,8 @@ package org.apache.kylin.metadata.measure.fixedlen;
import org.apache.kylin.metadata.model.DataType;
+import java.nio.ByteBuffer;
+
abstract public class FixedLenMeasureCodec<T> {
public static FixedLenMeasureCodec<?> get(DataType type) {
@@ -42,4 +44,6 @@ abstract public class FixedLenMeasureCodec<T> {
abstract public void write(T v, byte[] buf, int offset);
+ abstract public T read(ByteBuffer buffer);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
index 4014c21..37eb0be 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
@@ -22,6 +22,8 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.DataType;
+import java.nio.ByteBuffer;
+
public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
private static final int SIZE = 8;
@@ -106,4 +108,10 @@ public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
public void write(LongMutable v, byte[] buf, int offset) {
BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE);
}
+
+ @Override
+ public LongMutable read(ByteBuffer buffer) {
+ current.set(BytesUtil.readLong(buffer, SIZE));
+ return current;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 3a0049d..95cca0d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -199,26 +199,26 @@ public class EndpointAggregators {
* @param buffer byte buffer to get the metric data
* @return length of metric data
*/
- public int serializeMetricValues(MeasureAggregator[] aggrs, byte[] buffer) {
+ public int serializeMetricValues(MeasureAggregator[] aggrs, byte[] buffer, int offset) {
for (int i = 0; i < funcNames.length; i++) {
metricValues[i] = aggrs[i].getState();
}
- int metricBytesOffset = 0;
+ int metricBytesOffset = offset;
+ int length = 0;
for (int i = 0; i < measureSerializers.length; i++) {
measureSerializers[i].write(metricValues[i], buffer, metricBytesOffset);
metricBytesOffset += measureSerializers[i].getLength();
+ length += measureSerializers[i].getLength();
}
- return metricBytesOffset;
+ return length;
}
-
- public List<Object> deserializeMetricValues(byte[] metricBytes, int offset) {
+
+ public List<Object> deserializeMetricValues(ByteBuffer buffer) {
List<Object> ret = Lists.newArrayList();
- int metricBytesOffset = offset;
for (int i = 0; i < measureSerializers.length; i++) {
- measureSerializers[i].read(metricBytes, metricBytesOffset);
+ measureSerializers[i].read(buffer);
Object valueString = measureSerializers[i].getValue();
- metricBytesOffset += measureSerializers[i].getLength();
ret.add(valueString);
}
return ret;
@@ -279,4 +279,11 @@ public class EndpointAggregators {
}
+ public int getMeasureSerializeLength() {
+ int length = 0;
+ for (int i = 0; i < this.measureSerializers.length; ++i) {
+ length += this.measureSerializers[i].getLength();
+ }
+ return length;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 6d3ec4d..99db123 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -28,6 +29,7 @@ import java.util.Map;
import javax.annotation.Nullable;
+import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.hadoop.hbase.client.HConnection;
@@ -340,9 +342,8 @@ public class EndpointTupleIterator implements ITupleIterator {
byte[] columnsBytes = HBaseZeroCopyByteString.zeroCopyGetBytes(currentRow.getColumns());
this.tableRecord.setBytes(columnsBytes, 0, columnsBytes.length);
if (currentRow.hasMeasures()) {
- byte[] measuresBytes = HBaseZeroCopyByteString.zeroCopyGetBytes(currentRow.getMeasures());
-
- this.measureValues = pushedDownAggregators.deserializeMetricValues(measuresBytes, 0);
+ ByteBuffer buffer = currentRow.getMeasures().asReadOnlyByteBuffer();
+ this.measureValues = pushedDownAggregators.deserializeMetricValues(buffer);
}
index++;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b7cb41a8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index 6173241..e4360a2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -205,7 +205,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
RowKeyColumnIO rowKeyColumnIO = new RowKeyColumnIO(clearTextDictionary);
byte[] recordBuffer = new byte[recordInfo.getByteFormLen()];
- final byte[] buffer = new byte[CoprocessorConstants.METRIC_SERIALIZE_BUFFER_SIZE];
+ byte[] buffer = new byte[CoprocessorConstants.METRIC_SERIALIZE_BUFFER_SIZE];
int iteratedSliceCount = 0;
long latestSliceTs = Long.MIN_VALUE;
@@ -267,11 +267,18 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
logger.info("Iterated Slices count: " + iteratedSliceCount);
if (needAgg) {
+ int offset = 0;
+ int measureLength = aggregators.getMeasureSerializeLength();
for (Map.Entry<AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
AggrKey aggrKey = entry.getKey();
IIProtos.IIResponseInternal.IIRow.Builder rowBuilder = IIProtos.IIResponseInternal.IIRow.newBuilder().setColumns(HBaseZeroCopyByteString.wrap(aggrKey.get(), aggrKey.offset(), aggrKey.length()));
- int length = aggregators.serializeMetricValues(entry.getValue(), buffer);
- rowBuilder.setMeasures(HBaseZeroCopyByteString.wrap(buffer, 0, length));
+ if (offset + measureLength > buffer.length) {
+ buffer = new byte[CoprocessorConstants.METRIC_SERIALIZE_BUFFER_SIZE];
+ offset = 0;
+ }
+ int length = aggregators.serializeMetricValues(entry.getValue(), buffer, offset);
+ rowBuilder.setMeasures(HBaseZeroCopyByteString.wrap(buffer, offset, length));
+ offset += length;
responseBuilder.addRows(rowBuilder.build());
}
}
[31/31] incubator-kylin git commit: KYLIN-942 only check mem use at
query time
Posted by sh...@apache.org.
KYLIN-942 only check mem use at query time
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/91057748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/91057748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/91057748
Branch: refs/heads/KYLIN-1112
Commit: 91057748ce02c5ebb750f062253c927536300343
Parents: 42f1254
Author: honma <ho...@ebay.com>
Authored: Mon Nov 2 18:52:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/MemoryBudgetController.java | 2 +-
.../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java | 2 +-
.../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 5 +++--
.../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
4 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 2c26666..dc49b17 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,7 +57,7 @@ public class MemoryBudgetController {
public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
public static final int ONE_MB = 1024 * 1024;
- public static final int ONE_GB = 1024 * 1024 * 1024;
+ public static final long ONE_GB = 1024 * 1024 * 1024;
private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index d7aa50a..a179d70 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -343,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
- GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false);
+ GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + baseCuboidId);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 35c1210..cd2881e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner {
void aggregate(GTRecord r) {
if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) {
- if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
- throw new RuntimeException("AggregationCache exceed 1GB");
+ long estimated = estimatedMemSize();
+ if (estimated > 10 * MemoryBudgetController.ONE_GB) {
+ throw new RuntimeException("AggregationCache exceed 10GB, estimated size is: " + estimated);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index ba997ec..2c284c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -118,7 +118,7 @@ public class GTScanRequest {
}
public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
- return decorateScanner(scanner, true, true, true);
+ return decorateScanner(scanner, true, true, false);//by default do not check mem
}
/**
[27/31] incubator-kylin git commit: KYLIN-596 Support PowerBI desktop
(ver.2.28.4190.122)
Posted by sh...@apache.org.
KYLIN-596 Support PowerBI desktop (ver.2.28.4190.122)
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/276dfbee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/276dfbee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/276dfbee
Branch: refs/heads/KYLIN-1112
Commit: 276dfbee4e78771567b9ff35903da2efac4f377f
Parents: 4e161aa
Author: lidongsjtu <do...@ebay.com>
Authored: Wed Nov 4 10:05:53 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
odbc/Common/JDBCODBC.cpp | 8 ++---
odbc/Driver/KO_ATTR.CPP | 5 +++
odbc/Driver/KO_DESC.CPP | 2 +-
odbc/Driver/KO_FETCH.CPP | 8 +++--
odbc/Driver/KO_INFO.CPP | 72 ++++++++++++++++++++++++++++++++++++++++---
odbc/Driver/KO_UTILS.CPP | 2 +-
6 files changed, 84 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Common/JDBCODBC.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/JDBCODBC.cpp b/odbc/Common/JDBCODBC.cpp
index 8753fd2..a3b056e 100644
--- a/odbc/Common/JDBCODBC.cpp
+++ b/odbc/Common/JDBCODBC.cpp
@@ -68,13 +68,13 @@ ODBCTypes JDBC2ODBC ( int jtype ) {
break;
case JDBC_CHAR:
- //return ODBCTypes::ODBC_Char;
- return ODBCTypes::ODBC_WChar;//it's a unicode dirver
+ return ODBCTypes::ODBC_Char;
+ //return ODBCTypes::ODBC_WChar;//it's a unicode dirver
break;
case JDBC_VARCHAR:
- //return ODBCTypes::ODBC_VarChar;
- return ODBCTypes::ODBC_WChar;//it's a unicode dirver
+ return ODBCTypes::ODBC_VarChar;
+ //return ODBCTypes::ODBC_WChar;//it's a unicode dirver
break;
case 2000://"ANY" type in KYLIN
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Driver/KO_ATTR.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_ATTR.CPP b/odbc/Driver/KO_ATTR.CPP
index 7cfae45..3f21108 100644
--- a/odbc/Driver/KO_ATTR.CPP
+++ b/odbc/Driver/KO_ATTR.CPP
@@ -431,6 +431,11 @@ RETCODE SQL_API SQLSetStmtAttrW ( SQLHSTMT pStmt,
_SQLFreeDiag ( _DIAGSTMT ( pStmt ) );
switch ( pAttr ) {
+ case 1226:/* MS SQL Server Extension */
+ case 1227:
+ case 1228:
+ break;
+
case SQL_ATTR_APP_PARAM_DESC:
__ODBCPOPMSG ( _ODBCPopMsg ( "SQL_ATTR_APP_PARAM_DESC - Cannot set descriptor" ) );
return SQL_ERROR;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Driver/KO_DESC.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_DESC.CPP b/odbc/Driver/KO_DESC.CPP
index 3a92e26..b72deab 100644
--- a/odbc/Driver/KO_DESC.CPP
+++ b/odbc/Driver/KO_DESC.CPP
@@ -2477,7 +2477,7 @@ RETCODE SQL_API _SQLGetIRDItemField ( const pODBCIRD pDesc, const pIRDItem pDesc
case SQL_DESC_TYPE://1002
case SQL_DESC_CONCISE_TYPE://2
- * ( ( Long* ) pDataPtr ) = pDescItem->columnType;
+ * ( ( SQLLEN* ) pDataPtr ) = pDescItem->columnType;
break;
case SQL_DESC_DATETIME_INTERVAL_CODE://1007
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Driver/KO_FETCH.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_FETCH.CPP b/odbc/Driver/KO_FETCH.CPP
index 1053e32..39f0915 100644
--- a/odbc/Driver/KO_FETCH.CPP
+++ b/odbc/Driver/KO_FETCH.CPP
@@ -251,7 +251,10 @@ RETCODE SQL_API SQLColAttribute ( SQLHSTMT pStmt,
SQLSMALLINT* pDataSizePtr,
SQLLEN* pNumValuePtr ) {
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "SQLColAttributeW called, ColNum: %d, FldID: %d", pColNum, pFldID ) );
- return _SQLColAttribute_basic ( pStmt, pColNum, pFldID, pDataPtr, pDataSize, pDataSizePtr, pNumValuePtr, true );
+ RETCODE code = _SQLColAttribute_basic ( pStmt, pColNum, pFldID, pDataPtr, pDataSize, pDataSizePtr, pNumValuePtr,
+ true );
+ __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "the return code is %d", code ) );
+ return code;
}
#else
@@ -494,6 +497,7 @@ RETCODE SQL_API SQLBindCol ( SQLHSTMT pStmt,
ardcol = new ARDItem;
// reset
_SQLSetARDItemFieldsDefault ( ardcol, pColNum );
+
// set all values - bind
_SQLSetARDItemField ( ard, ardcol, pColNum, SQL_DESC_DATA_PTR, pDataPtr, -1 );
_SQLSetARDItemField ( ard, ardcol, pColNum, SQL_DESC_CONCISE_TYPE, ( void* ) pDataType, -1 );
@@ -949,7 +953,7 @@ RETCODE SQL_API _SQLFetchCol ( pODBCStmt pStmt,
// move both data and size ptr in the array
//TODO find out where the pARDCol->DataSize if set
tgtdataptr = ( void* ) ( ( ( Char* ) ( pARDCol->DataPtr ) ) + j + ( i * pARDCol->DataSize ) ); // use based on data type
- tgtsizeptr = ( Long* ) ( ( ( Char* ) ( pARDCol->SizePtr ) ) + j + ( i * sizeof ( Long ) ) );
+ tgtsizeptr = ( Long* ) ( ( ( Char* ) ( pARDCol->SizePtr ) ) + j + ( i * sizeof ( SQLLEN ) ) );
}
// PRECAUTION
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Driver/KO_INFO.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_INFO.CPP b/odbc/Driver/KO_INFO.CPP
index d8287b0..a90c1de 100644
--- a/odbc/Driver/KO_INFO.CPP
+++ b/odbc/Driver/KO_INFO.CPP
@@ -72,6 +72,31 @@ RETCODE SQL_API SQLGetInfoW ( SQLHDBC pConn,
break;
case SQL_CONVERT_FUNCTIONS ://48 called
+ case SQL_CONVERT_BIGINT:
+ case SQL_CONVERT_BINARY:
+ case SQL_CONVERT_BIT:
+ case SQL_CONVERT_CHAR:
+ case SQL_CONVERT_GUID:
+ case SQL_CONVERT_DATE:
+ case SQL_CONVERT_DECIMAL:
+ case SQL_CONVERT_DOUBLE:
+ case SQL_CONVERT_FLOAT:
+ case SQL_CONVERT_INTEGER:
+ case SQL_CONVERT_INTERVAL_YEAR_MONTH:
+ case SQL_CONVERT_INTERVAL_DAY_TIME:
+ case SQL_CONVERT_LONGVARBINARY:
+ case SQL_CONVERT_LONGVARCHAR:
+ case SQL_CONVERT_NUMERIC:
+ case SQL_CONVERT_REAL:
+ case SQL_CONVERT_SMALLINT:
+ case SQL_CONVERT_TIME:
+ case SQL_CONVERT_TIMESTAMP:
+ case SQL_CONVERT_TINYINT:
+ case SQL_CONVERT_VARBINARY:
+ case SQL_CONVERT_VARCHAR:
+ case SQL_CONVERT_WCHAR:
+ case SQL_CONVERT_WLONGVARCHAR:
+ case SQL_CONVERT_WVARCHAR:
break;
case SQL_MAX_IDENTIFIER_LEN : //10005 called
@@ -210,12 +235,15 @@ RETCODE SQL_API SQLGetInfoW ( SQLHDBC pConn,
break;
+ case SQL_DTC_TRANSITION_COST: // 1750
+ break;
+
case SQL_DATA_SOURCE_READ_ONLY: // 25
//MessageBox ( GetDesktopWindow(), "SQL_DATA_SOURCE_READ_ONLY", "SQLGetInfo", MB_OK );
_SQLCopyWCharData ( _DIAGCONN ( pConn ), pInfoValuePtr, pBufferLength, pStringLengthPtr, 16, "N", -1 );
break;
- case SQL_DRIVER_NAME: // 6 //called by tb
+ case SQL_DRIVER_NAME: // 6
//MessageBox ( GetDesktopWindow(), "SQL_DRIVER_NAME", "SQLGetInfo", MB_OK );
_SQLCopyWCharData ( _DIAGCONN ( pConn ), pInfoValuePtr, pBufferLength, pStringLengthPtr, 16, "driver.DLL", -1 );
break;
@@ -275,11 +303,11 @@ RETCODE SQL_API SQLGetInfoW ( SQLHDBC pConn,
_SQLCopyWCharData ( _DIAGCONN ( pConn ), pInfoValuePtr, pBufferLength, pStringLengthPtr, 16, "\"", -1 );
break;
- case SQL_MAX_SCHEMA_NAME_LEN:
+ case SQL_MAX_SCHEMA_NAME_LEN: // 32
* ( ( short* ) pInfoValuePtr ) = 128;
break;
- case SQL_USER_NAME:
+ case SQL_USER_NAME: // 47
_SQLCopyWCharData ( _DIAGCONN ( pConn ), pInfoValuePtr, pBufferLength, pStringLengthPtr, 16, "default", -1 );
break;
@@ -399,6 +427,9 @@ RETCODE SQL_API SQLGetInfoW ( SQLHDBC pConn,
//__ODBCPOPMSG(_ODBCPopMsg("SQLGetInfo, SQL_CATALOG_LOCATION" ));
break;
+ case SQL_CATALOG_NAME: // 10003
+ break;
+
case SQL_CONCAT_NULL_BEHAVIOR: // 22
* ( ( Word* ) pInfoValuePtr ) = SQL_CB_NULL;
//__ODBCPOPMSG(_ODBCPopMsg("SQLGetInfo, SQL_CONCAT_NULL_BEHAVIOR" ));
@@ -612,6 +643,31 @@ RETCODE SQL_API SQLGetInfo ( SQLHDBC pConn,
break;
case SQL_CONVERT_FUNCTIONS ://48 called
+ case SQL_CONVERT_BIGINT:
+ case SQL_CONVERT_BINARY:
+ case SQL_CONVERT_BIT:
+ case SQL_CONVERT_CHAR:
+ case SQL_CONVERT_GUID:
+ case SQL_CONVERT_DATE:
+ case SQL_CONVERT_DECIMAL:
+ case SQL_CONVERT_DOUBLE:
+ case SQL_CONVERT_FLOAT:
+ case SQL_CONVERT_INTEGER:
+ case SQL_CONVERT_INTERVAL_YEAR_MONTH:
+ case SQL_CONVERT_INTERVAL_DAY_TIME:
+ case SQL_CONVERT_LONGVARBINARY:
+ case SQL_CONVERT_LONGVARCHAR:
+ case SQL_CONVERT_NUMERIC:
+ case SQL_CONVERT_REAL:
+ case SQL_CONVERT_SMALLINT:
+ case SQL_CONVERT_TIME:
+ case SQL_CONVERT_TIMESTAMP:
+ case SQL_CONVERT_TINYINT:
+ case SQL_CONVERT_VARBINARY:
+ case SQL_CONVERT_VARCHAR:
+ case SQL_CONVERT_WCHAR:
+ case SQL_CONVERT_WLONGVARCHAR:
+ case SQL_CONVERT_WVARCHAR:
break;
case SQL_MAX_IDENTIFIER_LEN : //10005 called
@@ -621,7 +677,7 @@ RETCODE SQL_API SQLGetInfo ( SQLHDBC pConn,
break;
case SQL_SQL_CONFORMANCE: //118 called
- break;
+ break;
/*
An SQLUINTEGER bitmask enumerating the SQL-92 datetime literals supported by the data source. Note that these are the datetime literals listed in the SQL-92 specification and are separate from the datetime literal escape clauses defined by ODBC. For more information about the ODBC datetime literal escape clauses, see Date, Time, and Timestamp Literals.
@@ -748,6 +804,9 @@ RETCODE SQL_API SQLGetInfo ( SQLHDBC pConn,
if ( pInfoValuePtr ) { * ( ( Word* ) pInfoValuePtr ) = SQL_OAC_NONE; } // for MS Access
break;
+
+ case SQL_DTC_TRANSITION_COST: // 1750
+ break;
case SQL_DATA_SOURCE_READ_ONLY: // 25
//MessageBox ( GetDesktopWindow(), "SQL_DATA_SOURCE_READ_ONLY", "SQLGetInfo", MB_OK );
@@ -937,7 +996,10 @@ RETCODE SQL_API SQLGetInfo ( SQLHDBC pConn,
* ( ( Word* ) pInfoValuePtr ) = SQL_CL_START;
//__ODBCPOPMSG(_ODBCPopMsg("SQLGetInfo, SQL_CATALOG_LOCATION" ));
break;
-
+
+ case SQL_CATALOG_NAME: // 10003
+ break;
+
case SQL_CONCAT_NULL_BEHAVIOR: // 22
* ( ( Word* ) pInfoValuePtr ) = SQL_CB_NULL;
//__ODBCPOPMSG(_ODBCPopMsg("SQLGetInfo, SQL_CONCAT_NULL_BEHAVIOR" ));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/276dfbee/odbc/Driver/KO_UTILS.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_UTILS.CPP b/odbc/Driver/KO_UTILS.CPP
index ea7ccf7..d46deff 100644
--- a/odbc/Driver/KO_UTILS.CPP
+++ b/odbc/Driver/KO_UTILS.CPP
@@ -274,7 +274,7 @@ RETCODE SQL_API _SQLCopyWCharDataW ( pODBCDiag pDiag, void* pTgtDataPtr, Long pD
else
{ * ( ( Long* ) pSizePtr ) = pPtrSizeBuf; }
- __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "pSizePtr is set to %d", n ) );
+ __ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "pSizePtr %d is set to %d", pSizePtr, * ( ( SQLLEN* ) pSizePtr ) ) );
}
// check if src data but no size holder
[09/31] incubator-kylin git commit: fix CI,
ignore check for dictionary on TopN measure when building empty
segment
Posted by sh...@apache.org.
fix CI, ignore check for dictionary on TopN measure when building empty segment
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/279226f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/279226f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/279226f8
Branch: refs/heads/KYLIN-1112
Commit: 279226f87090b04e4b30a0e65b868e9d4648b084
Parents: 5149570
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Oct 29 17:24:45 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:16 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/279226f8/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index 3b68d47..f0ee372 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.cube.inmemcubing;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
@@ -47,7 +46,8 @@ public final class InMemCubeBuilderUtils {
TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
@SuppressWarnings("unchecked")
Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(displayCol);
- result.put(displayColIdx, Preconditions.checkNotNull(dictionary));
+ //Preconditions.checkNotNull(dictionary);//FIXME disable check since dictionary is null when building empty segment
+ result.put(displayColIdx, dictionary);
}
}
return result;
[05/31] incubator-kylin git commit: KYLIN-1094 use Kyro Serializer
for SparkCubing
Posted by sh...@apache.org.
KYLIN-1094 use Kyro Serializer for SparkCubing
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/730739f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/730739f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/730739f7
Branch: refs/heads/KYLIN-1112
Commit: 730739f70797f296cb9cb0a1cb94c8d7649f769e
Parents: 6ab480c
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Oct 29 11:30:06 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:15 2015 +0800
----------------------------------------------------------------------
commit_SHA1 | 16 --
.../cube/inmemcubing/InMemCubeBuilder.java | 159 ++-----------------
.../InMemCubeBuilderInputConverter.java | 146 +++++++++++++++++
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 85 ++++++++++
.../apache/kylin/engine/spark/SparkCubing.java | 86 ++++++++--
5 files changed, 316 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/730739f7/commit_SHA1
----------------------------------------------------------------------
diff --git a/commit_SHA1 b/commit_SHA1
deleted file mode 100644
index d2f3970..0000000
--- a/commit_SHA1
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-b66c25803a2f976cca067148278dbe7d7b0d79ef
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/730739f7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 7d943a3..84abd47 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -17,14 +17,7 @@
package org.apache.kylin.cube.inmemcubing;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -32,11 +25,12 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.*;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.MemoryBudgetController;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
-import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.dict.Dictionary;
@@ -50,14 +44,12 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.measure.DoubleMutable;
import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
/**
* Build a cube (many cuboids) in memory. Calculating multiple cuboids at the same time as long as memory permits.
@@ -87,7 +79,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private CuboidResult baseResult;
private Object[] totalSumForSanityCheck;
private ICuboidCollector resultCollector;
- private Map<Integer, Dictionary<String>> topNDisplayColDictMap;
public InMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
super(cubeDesc, dictionaryMap);
@@ -100,35 +91,16 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
- Map<String, Integer> measureIndexMap = Maps.newHashMap();
List<String> metricsAggrFuncsList = Lists.newArrayList();
for (int i = 0; i < measureCount; i++) {
MeasureDesc measureDesc = measureDescs[i];
metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
- measureIndexMap.put(measureDesc.getName(), i);
}
this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
-
- initTopNDisplayColDictionaryMap();
}
- private void initTopNDisplayColDictionaryMap() {
- topNDisplayColDictMap = Maps.newHashMap();
- for (int measureIdx = 0; measureIdx < cubeDesc.getMeasures().size(); measureIdx++) {
- MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
- FunctionDesc func = measureDesc.getFunction();
- if (func.isTopN()) {
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
- int displayColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
- @SuppressWarnings("unchecked")
- Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(displayCol);
- assert dictionary != null;
- topNDisplayColDictMap.put(displayColIdx, dictionary);
- }
- }
- }
+
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
GTInfo info = CubeGridTable.newGTInfo(cubeDesc, cuboidID, dictionaryMap);
@@ -142,14 +114,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
return gridTable;
}
- private Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(long cuboidId) {
- BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
- BitSet dimension = new BitSet();
- dimension.set(0, bitSet.cardinality());
- BitSet metrics = new BitSet();
- metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.measureCount);
- return new Pair<ImmutableBitSet, ImmutableBitSet>(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
- }
@Override
public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
@@ -376,7 +340,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
int mbBefore = getSystemAvailMB();
int mbAfter = 0;
- Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId);
+ Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
@@ -442,27 +406,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private CuboidResult aggregateCuboid(CuboidResult parent, long cuboidId) throws IOException {
- Pair<ImmutableBitSet, ImmutableBitSet> columnBitSets = getDimensionAndMetricColumnBitSet(parent.cuboidId);
- ImmutableBitSet parentDimensions = columnBitSets.getFirst();
- ImmutableBitSet measureColumns = columnBitSets.getSecond();
- ImmutableBitSet childDimensions = parentDimensions;
-
- long mask = Long.highestOneBit(parent.cuboidId);
- long childCuboidId = cuboidId;
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parent.cuboidId);
- int index = 0;
- for (int i = 0; i < parentCuboidIdActualLength; i++) {
- if ((mask & parent.cuboidId) > 0) {
- if ((mask & childCuboidId) == 0) {
- // this dim will be aggregated
- childDimensions = childDimensions.set(index, false);
- }
- index++;
- }
- mask = mask >> 1;
- }
-
- return scanAndAggregateGridTable(parent.table, parent.cuboidId, cuboidId, childDimensions, measureColumns);
+ final Pair<ImmutableBitSet, ImmutableBitSet> allNeededColumns = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(parent.cuboidId, cuboidId, measureCount);
+ return scanAndAggregateGridTable(parent.table, parent.cuboidId, cuboidId, allNeededColumns.getFirst(), allNeededColumns.getSecond());
}
private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException {
@@ -557,12 +502,15 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
GTInfo info;
GTRecord record;
BlockingQueue<List<String>> input;
- ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ final InMemCubeBuilderInputConverter inMemCubeBuilderInputConverter;
public InputConverter(GTInfo info, BlockingQueue<List<String>> input) {
this.info = info;
this.input = input;
this.record = new GTRecord(info);
+ this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc,
+ InMemCubeBuilderUtils.createTopNDisplayColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap),
+ info);
}
@Override
@@ -586,7 +534,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
if (currentObject.size() == 0)
throw new IllegalStateException();
- buildGTRecord(currentObject, record);
+ inMemCubeBuilderInputConverter.convert(currentObject, record);
return record;
}
@@ -611,86 +559,5 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
return 0;
}
- private void buildGTRecord(List<String> row, GTRecord record) {
- Object[] dimensions = buildKey(row);
- Object[] metricsValues = buildValue(row);
- Object[] recordValues = new Object[dimensions.length + metricsValues.length];
- System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
- System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length);
- record.setValues(recordValues);
- }
-
- private Object[] buildKey(List<String> row) {
- int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
- Object[] key = new Object[keySize];
-
- for (int i = 0; i < keySize; i++) {
- key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
- }
-
- return key;
- }
-
- private Object[] buildValue(List<String> row) {
-
- Object[] values = new Object[measureCount];
- MeasureDesc measureDesc = null;
-
- for (int i = 0; i < measureCount; i++) {
- measureDesc = measureDescs[i];
-
- Object value = null;
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
- FunctionDesc function = cubeDesc.getMeasures().get(i).getFunction();
- if (flatTableIdx == null) {
- value = measureCodec.getSerializer(i).valueOf(measureDesc.getFunction().getParameter().getValue());
- } else if (function.isCount() || function.isHolisticCountDistinct()) {
- // note for holistic count distinct, this value will be ignored
- value = ONE;
- } else if (function.isTopN()) {
- // encode the key column with dict, and get the counter column;
- int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
- Dictionary<String> displayColDict = topNDisplayColDictMap.get(keyColIndex);
- int keyColEncoded = displayColDict.getIdFromValue(row.get(keyColIndex));
- valueBuf.clear();
- valueBuf.putInt(displayColDict.getSizeOfId());
- valueBuf.putInt(keyColEncoded);
- if (flatTableIdx.length == 1) {
- // only displayCol, use 1.0 as counter
- valueBuf.putDouble(1.0);
- } else {
- // get the counter column value
- valueBuf.putDouble(Double.valueOf(row.get(flatTableIdx[0])));
- }
-
- value = measureCodec.getSerializer(i).valueOf(valueBuf.array());
-
- } else if (flatTableIdx.length == 1) {
- value = measureCodec.getSerializer(i).valueOf(toBytes(row.get(flatTableIdx[0])));
- } else {
-
- byte[] result = null;
- for (int x = 0; x < flatTableIdx.length; x++) {
- byte[] split = toBytes(row.get(flatTableIdx[x]));
- if (result == null) {
- result = Arrays.copyOf(split, split.length);
- } else {
- byte[] newResult = new byte[result.length + split.length];
- System.arraycopy(result, 0, newResult, 0, result.length);
- System.arraycopy(split, 0, newResult, result.length, split.length);
- result = newResult;
- }
- }
- value = measureCodec.getSerializer(i).valueOf(result);
- }
- values[i] = value;
- }
- return values;
- }
-
- private byte[] toBytes(String v) {
- return v == null ? null : Bytes.toBytes(v);
- }
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/730739f7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
new file mode 100644
index 0000000..d9099ce
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.cube.inmemcubing;
+
+import com.google.common.base.Preconditions;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.measure.LongMutable;
+import org.apache.kylin.metadata.measure.MeasureCodec;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ */
+public class InMemCubeBuilderInputConverter {
+
+ private static final LongMutable ONE = new LongMutable(1l);
+
+ private final CubeDesc cubeDesc;
+ private final CubeJoinedFlatTableDesc intermediateTableDesc;
+ private final MeasureDesc[] measureDescs;
+ private final MeasureCodec measureCodec;
+ private final int measureCount;
+ private final ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ private final Map<Integer, Dictionary<String>> topNDisplayColDictMap;
+ private final GTInfo gtInfo;
+
+
+ public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<Integer, Dictionary<String>> topNDisplayColDictMap, GTInfo gtInfo) {
+ this.cubeDesc = cubeDesc;
+ this.gtInfo = gtInfo;
+ this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
+ this.measureCount = cubeDesc.getMeasures().size();
+ this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
+ this.measureCodec = new MeasureCodec(cubeDesc.getMeasures());
+ this.topNDisplayColDictMap = Preconditions.checkNotNull(topNDisplayColDictMap, "topNDisplayColDictMap cannot be null");
+ }
+
+ public final GTRecord convert(List<String> row) {
+ final GTRecord record = new GTRecord(gtInfo);
+ convert(row, record);
+ return record;
+ }
+
+ public final void convert(List<String> row, GTRecord record) {
+ Object[] dimensions = buildKey(row);
+ Object[] metricsValues = buildValue(row);
+ Object[] recordValues = new Object[dimensions.length + metricsValues.length];
+ System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
+ System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length);
+ record.setValues(recordValues);
+ }
+
+ private Object[] buildKey(List<String> row) {
+ int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
+ Object[] key = new Object[keySize];
+
+ for (int i = 0; i < keySize; i++) {
+ key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+ }
+
+ return key;
+ }
+
+ private Object[] buildValue(List<String> row) {
+
+ Object[] values = new Object[measureCount];
+ for (int i = 0; i < measureCount; i++) {
+ MeasureDesc measureDesc = measureDescs[i];
+ int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
+ FunctionDesc function = cubeDesc.getMeasures().get(i).getFunction();
+ if (flatTableIdx == null) {
+ values[i] = measureCodec.getSerializer(i).valueOf(measureDesc.getFunction().getParameter().getValue());
+ } else if (function.isCount() || function.isHolisticCountDistinct()) {
+ // note for holistic count distinct, this value will be ignored
+ values[i] = ONE;
+ } else if (function.isTopN()) {
+ // encode the key column with dict, and get the counter column;
+ int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
+ Dictionary<String> displayColDict = topNDisplayColDictMap.get(keyColIndex);
+ int keyColEncoded = displayColDict.getIdFromValue(row.get(keyColIndex));
+ valueBuf.clear();
+ valueBuf.putInt(displayColDict.getSizeOfId());
+ valueBuf.putInt(keyColEncoded);
+ if (flatTableIdx.length == 1) {
+ // only displayCol, use 1.0 as counter
+ valueBuf.putDouble(1.0);
+ } else {
+ // get the counter column value
+ valueBuf.putDouble(Double.valueOf(row.get(flatTableIdx[0])));
+ }
+
+ values[i] = measureCodec.getSerializer(i).valueOf(valueBuf.array());
+
+ } else if (flatTableIdx.length == 1) {
+ values[i] = measureCodec.getSerializer(i).valueOf(toBytes(row.get(flatTableIdx[0])));
+ } else {
+
+ byte[] result = null;
+ for (int x = 0; x < flatTableIdx.length; x++) {
+ byte[] split = toBytes(row.get(flatTableIdx[x]));
+ if (result == null) {
+ result = Arrays.copyOf(split, split.length);
+ } else {
+ byte[] newResult = new byte[result.length + split.length];
+ System.arraycopy(result, 0, newResult, 0, result.length);
+ System.arraycopy(split, 0, newResult, result.length, split.length);
+ result = newResult;
+ }
+ }
+ values[i] = measureCodec.getSerializer(i).valueOf(result);
+ }
+ }
+ return values;
+ }
+
+ private byte[] toBytes(String v) {
+ return v == null ? null : Bytes.toBytes(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/730739f7/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
new file mode 100644
index 0000000..3b68d47
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.cube.inmemcubing;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ */
+public final class InMemCubeBuilderUtils {
+
+ public static final HashMap<Integer, Dictionary<String>> createTopNDisplayColDictionaryMap(CubeDesc cubeDesc, CubeJoinedFlatTableDesc intermediateTableDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ HashMap<Integer, Dictionary<String>> result = Maps.newHashMap();
+ for (int measureIdx = 0; measureIdx < cubeDesc.getMeasures().size(); measureIdx++) {
+ MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
+ FunctionDesc func = measureDesc.getFunction();
+ if (func.isTopN()) {
+ int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
+ int displayColIdx = flatTableIdx[flatTableIdx.length - 1];
+ TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
+ @SuppressWarnings("unchecked")
+ Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(displayCol);
+ result.put(displayColIdx, Preconditions.checkNotNull(dictionary));
+ }
+ }
+ return result;
+ }
+
+ public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
+ BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
+ BitSet dimension = new BitSet();
+ dimension.set(0, bitSet.cardinality());
+ BitSet metrics = new BitSet();
+ metrics.set(bitSet.cardinality(), bitSet.cardinality() + measureCount);
+ return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
+ }
+
+ public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long baseCuboidId, final long childCuboidId, final int measureCount) {
+ final Pair<ImmutableBitSet, ImmutableBitSet> parentDimensionAndMetricColumnBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
+ ImmutableBitSet parentDimensions = parentDimensionAndMetricColumnBitSet.getFirst();
+ ImmutableBitSet measureColumns = parentDimensionAndMetricColumnBitSet.getSecond();
+ ImmutableBitSet childDimensions = parentDimensions;
+ long mask = Long.highestOneBit(baseCuboidId);
+ long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
+ int index = 0;
+ for (int i = 0; i < parentCuboidIdActualLength; i++) {
+ if ((mask & baseCuboidId) > 0) {
+ if ((mask & childCuboidId) == 0) {
+ // this dim will be aggregated
+ childDimensions = childDimensions.set(index, false);
+ }
+ index++;
+ }
+ mask = mask >> 1;
+ }
+ return Pair.newPair(childDimensions, measureColumns);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/730739f7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 0e5081e..e4fcc16 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -17,9 +17,8 @@
*/
package org.apache.kylin.engine.spark;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import com.google.common.base.*;
+import com.google.common.collect.*;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
@@ -28,6 +27,7 @@ import com.google.common.primitives.UnsignedBytes;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -54,6 +54,7 @@ import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.*;
import org.apache.kylin.cube.util.CubingUtils;
+import org.apache.kylin.dict.*;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -78,15 +79,18 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.*;
+import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.hive.HiveContext;
-
+import org.reflections.Reflections;
import scala.Tuple2;
+import javax.annotation.Nullable;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Executors;
@@ -143,12 +147,12 @@ public class SparkCubing extends AbstractApplication {
final CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
final String[] columns = intermediateTable.columns();
- long start = System.currentTimeMillis();
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap();
final CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
final List<TblColRef> baseCuboidColumn = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
- RowKeyDesc rowKey = cubeDesc.getRowkey();
+ final long start = System.currentTimeMillis();
+ final RowKeyDesc rowKey = cubeDesc.getRowkey();
for (int i = 0; i < baseCuboidColumn.size(); i++) {
TblColRef col = baseCuboidColumn.get(i);
if (!rowKey.isUseDictionary(col)) {
@@ -217,12 +221,13 @@ public class SparkCubing extends AbstractApplication {
}
CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
-
+
final int[] rowKeyColumnIndexes = flatTableDesc.getRowKeyColumnIndexes();
final int nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-
final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
+ final ByteArray[] row_hashcodes = new ByteArray[nRowKey];
+
for (Long cuboidId : allCuboidIds) {
BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
Integer[] cuboidBitSet = new Integer[bitSet.cardinality()];
@@ -238,16 +243,15 @@ public class SparkCubing extends AbstractApplication {
}
allCuboidsBitSet.put(cuboidId, cuboidBitSet);
}
- final ByteArray[] row_hashcodes = new ByteArray[nRowKey];
for (int i = 0; i < nRowKey; ++i) {
row_hashcodes[i] = new ByteArray();
}
-
+
final HashMap<Long, HyperLogLogPlusCounter> samplingResult = rowJavaRDD.aggregate(zeroValue,
new Function2<HashMap<Long, HyperLogLogPlusCounter>,
List<String>,
HashMap<Long, HyperLogLogPlusCounter>>() {
-
+
final HashFunction hashFunction = Hashing.murmur3_32();
@Override
@@ -319,7 +323,7 @@ public class SparkCubing extends AbstractApplication {
}
}
}
-
+
for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
if (measureDesc.getFunction().isTopN()) {
List<TblColRef> colRefs = measureDesc.getFunction().getParameter().getColRefs();
@@ -328,7 +332,7 @@ public class SparkCubing extends AbstractApplication {
}
}
-
+
final JavaPairRDD<byte[], byte[]> javaPairRDD = javaRDD.glom().mapPartitionsToPair(new PairFlatMapFunction<Iterator<List<List<String>>>, byte[], byte[]>() {
@Override
@@ -375,7 +379,7 @@ public class SparkCubing extends AbstractApplication {
writeToHFile2(javaPairRDD, dataTypes, measureSize, aggs, splitKeys, conf, url);
return url;
}
-
+
private void writeToHFile2(final JavaPairRDD<byte[], byte[]> javaPairRDD, final String[] dataTypes, final int measureSize, final MeasureAggregators aggs, final byte[][] splitKeys, final Configuration conf, final String hFileLocation) {
javaPairRDD.repartitionAndSortWithinPartitions(new Partitioner() {
@Override
@@ -505,13 +509,67 @@ public class SparkCubing extends AbstractApplication {
}
}
+ private Collection<String> getKyroClasses() {
+ Set<Class> kyroClasses = Sets.newHashSet();
+ kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
+ kyroClasses.add(HashMap.class);
+ kyroClasses.add(org.apache.spark.sql.Row[].class);
+ kyroClasses.add(org.apache.spark.sql.Row.class);
+ kyroClasses.add(org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema.class);
+ kyroClasses.add(org.apache.spark.sql.types.StructType.class);
+ kyroClasses.add(org.apache.spark.sql.types.StructField[].class);
+ kyroClasses.add(org.apache.spark.sql.types.StructField.class);
+ kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
+ kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
+ kyroClasses.add(Object[].class);
+ kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
+ kyroClasses.add(Hashing.murmur3_128().getClass());
+ kyroClasses.add(org.apache.spark.sql.columnar.CachedBatch.class);
+ kyroClasses.add(byte[][].class);
+ kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
+ kyroClasses.add(scala.math.BigDecimal.class);
+ kyroClasses.add(java.math.BigDecimal.class);
+ kyroClasses.add(java.math.MathContext.class);
+ kyroClasses.add(java.math.RoundingMode.class);
+ kyroClasses.add(java.util.ArrayList.class);
+ kyroClasses.add(java.util.LinkedList.class);
+
+
+ ArrayList<String> result = Lists.newArrayList();
+ for (Class kyroClass : kyroClasses) {
+ result.add(kyroClass.getName());
+ }
+ result.add("scala.collection.immutable.Map$EmptyMap$");
+ result.add("org.apache.spark.sql.catalyst.expressions.GenericInternalRow");
+ result.add("org.apache.spark.unsafe.types.UTF8String");
+ return result;
+ }
+
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
final String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_PATH);
SparkConf conf = new SparkConf().setAppName("Simple Application");
+ //memory conf
conf.set("spark.executor.memory", "6g");
conf.set("spark.storage.memoryFraction", "0.3");
+ //serialization conf
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ conf.set("spark.kryo.registrationRequired", "true");
+ final Iterable<String> allClasses = Iterables.filter(
+ Iterables.concat(Lists.newArrayList(conf.get("spark.kryo.classesToRegister", "").split(",")), getKyroClasses()),
+ new Predicate<String>() {
+ @Override
+ public boolean apply(@Nullable String input) {
+ return input != null && input.trim().length() > 0;
+ }
+ });
+ System.out.println("kyro classes:" + allClasses.toString());
+ conf.set("spark.kryo.classesToRegister", StringUtils.join(allClasses, ","));
+
JavaSparkContext sc = new JavaSparkContext(conf);
HiveContext sqlContext = new HiveContext(sc.sc());
final DataFrame intermediateTable = sqlContext.sql("select * from " + hiveTable);
[30/31] incubator-kylin git commit: KYLIN-942 fix bug in DebugTomcat
pom and Backdoor toggles
Posted by sh...@apache.org.
KYLIN-942 fix bug in DebugTomcat pom and Backdoor toggles
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b0b43cfc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b0b43cfc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b0b43cfc
Branch: refs/heads/KYLIN-1112
Commit: b0b43cfcfa21f01f2284c1552bc5f1fd040b5401
Parents: 9105774
Author: honma <ho...@ebay.com>
Authored: Tue Nov 3 11:34:43 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Nov 5 09:53:17 2015 +0800
----------------------------------------------------------------------
server/pom.xml | 4 ++++
.../kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 9 ++-------
2 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0b43cfc/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 96d874a..f2f9e32 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -108,6 +108,10 @@
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0b43cfc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 77a5db5..6ad30e2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -161,6 +161,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
final AtomicInteger totalScannedCount = new AtomicInteger(0);
+ final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
+ logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
for (int i = 0; i < rawScans.size(); ++i) {
final int shardIndex = i;
@@ -176,13 +178,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
- // debug/profiling purpose
- String toggle = BackdoorToggles.getCoprocessorBehavior();
- if (toggle == null) {
- toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
- } else {
- logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
- }
builder.setBehavior(toggle);
Collection<CubeVisitProtos.CubeVisitResponse> results;