You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/09/11 06:05:57 UTC
[01/43] kylin git commit: KYLIN-1698 backend done,
pending GUI [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/v1.5.4-release2 2292647b2 -> c59d63de6 (forced update)
KYLIN-1698 backend done, pending GUI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a20d709d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a20d709d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a20d709d
Branch: refs/heads/v1.5.4-release2
Commit: a20d709d27da9060d63ccb1c14902235a3aedec2
Parents: 16de8ba
Author: Li Yang <li...@apache.org>
Authored: Fri Sep 2 15:49:34 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Sep 2 15:49:48 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/datatype/DataType.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 20 +++++++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a20d709d/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 032fba2..c981f46 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -63,7 +63,7 @@ public class DataType implements Serializable {
register("any", "char", "varchar", "string", //
"boolean", "byte", "binary", //
"int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
- "int4", "long8", //
+ "int4", "long8", // for test only
"float", "real", "double", "decimal", "numeric", //
"date", "time", "datetime", "timestamp", //
InnerDataTypeEnum.LITERAL.getDataType(), InnerDataTypeEnum.DERIVED.getDataType());
http://git-wip-us.apache.org/repos/asf/kylin/blob/a20d709d/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 3f265e2..8d3979e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -90,6 +91,11 @@ public class PartitionDesc {
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
+ public boolean partitionColumnIsTimeMillis() {
+ DataType type = partitionDateColumnRef.getType();
+ return type.isBigInt();
+ }
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -166,7 +172,9 @@ public class PartitionDesc {
String partitionDateColumnName = partDesc.getPartitionDateColumn();
String partitionTimeColumnName = partDesc.getPartitionTimeColumn();
- if (partitionDateColumnName != null && partitionTimeColumnName == null) {
+ if (partDesc.partitionColumnIsTimeMillis()) {
+ buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias);
+ } else if (partitionDateColumnName != null && partitionTimeColumnName == null) {
buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), tableAlias);
} else if (partitionDateColumnName == null && partitionTimeColumnName != null) {
buildSingleColumnRangeCondition(builder, partitionTimeColumnName, startInclusive, endExclusive, partDesc.getPartitionTimeFormat(), tableAlias);
@@ -190,6 +198,15 @@ public class PartitionDesc {
return columnName;
}
+ private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
+ if (startInclusive > 0) {
+ builder.append(partitionColumnName + " >= " + startInclusive);
+ builder.append(" AND ");
+ }
+ builder.append(partitionColumnName + " < " + endExclusive);
+ }
+
private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, Map<String, String> tableAlias) {
partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
if (startInclusive > 0) {
@@ -272,4 +289,5 @@ public class PartitionDesc {
newPartDesc.setPartitionDateStart(partitionDesc.getPartitionDateStart());
return newPartDesc;
}
+
}
[14/43] kylin git commit: KYLIN-1926 check pk,
fk type when edit model join relation
Posted by sh...@apache.org.
KYLIN-1926 check pk,fk type when edit model join relation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f12a5e77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f12a5e77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f12a5e77
Branch: refs/heads/v1.5.4-release2
Commit: f12a5e77e81d88a89aa700dd2dccba7f2e0814ae
Parents: 4ede67e
Author: Jason <ji...@163.com>
Authored: Wed Sep 7 11:39:08 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 11:39:28 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/data_model.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f12a5e77/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 662d0d9..ace977f 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -173,7 +173,7 @@
</button>
</div>
<div class="space-4"></div>
- <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}],{{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}].</b></small>
+ <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}], {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}]</b></small>
</div>
</div>
</div>
[39/43] kylin git commit: minor update hcatalog folder for emr
Posted by sh...@apache.org.
minor update hcatalog folder for emr
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3450c0f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3450c0f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3450c0f9
Branch: refs/heads/v1.5.4-release2
Commit: 3450c0f984933d40cc60a604813a310accdfaded
Parents: 56136ed
Author: shaofengshi <sh...@apache.org>
Authored: Sat Sep 10 17:52:56 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 18:00:08 2016 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3450c0f9/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 539a45b..71c2fe6 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -86,9 +86,9 @@ then
hcatalog_home=${hadoop_home}/hive/hcatalog
elif [ -d "${hive_home}/hcatalog" ]; then
hcatalog_home=${hive_home}/hcatalog
- elif [ -n is_aws ] && [ -d "/usr/lib/oozie/lib" ]; then
- # special handling for Amazon EMR, where hcat libs are under oozie!?
- hcatalog_home=/usr/lib/oozie/lib
+ elif [ -n is_aws ] && [ -d "/usr/lib/hive-hcatalog" ]; then
+ # special handling for Amazon EMR
+ hcatalog_home=/usr/lib/hive-hcatalog
else
echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
exit 1
[06/43] kylin git commit: KYLIN-1992 Clear ThreadLocal Contexts when
query failed before scaning HBase
Posted by sh...@apache.org.
KYLIN-1992 Clear ThreadLocal Contexts when query failed before scaning HBase
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/663820f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/663820f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/663820f1
Branch: refs/heads/v1.5.4-release2
Commit: 663820f1a267c9ff156ffbe018ac1d95a1ede7cc
Parents: 67bcec2
Author: kangkaisen <ka...@live.com>
Authored: Thu Sep 1 16:02:42 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Sep 4 21:10:51 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/663820f1/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 3acaeb8..df296cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -316,6 +316,8 @@ public class QueryService extends BasicService {
parameters.put(OLAPContext.PRM_USER_AUTHEN_INFO, userInfo);
parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
OLAPContext.setParameters(parameters);
+ // force clear the query context before a new query
+ OLAPContext.clearThreadLocalContexts();
return execute(correctedSql, sqlRequest);
[36/43] kylin git commit: minor, fix job start/end in diagnosis
Posted by sh...@apache.org.
minor, fix job start/end in diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7cbf673
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7cbf673
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7cbf673
Branch: refs/heads/v1.5.4-release2
Commit: d7cbf6732a9571007de61dd492ed4d7559cbf9ac
Parents: d7a3fdf
Author: lidongsjtu <li...@apache.org>
Authored: Sat Sep 10 15:23:43 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sat Sep 10 16:09:19 2016 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7cbf673/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
index 086a84f..ef77c6a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
@@ -133,6 +133,8 @@ public class JobInstanceExtractor extends AbstractInfoExtractor {
result.setType(CubeBuildTypeEnum.BUILD);
result.setStatus(parseToJobStatus(output.getState()));
result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setExecStartTime(AbstractExecutable.getStartTime(output));
+ result.setExecEndTime(AbstractExecutable.getEndTime(output));
result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output)) / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
[19/43] kylin git commit: KYLIN-1789 fix wrong test cube metadata
Posted by sh...@apache.org.
KYLIN-1789 fix wrong test cube metadata
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc7281b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc7281b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc7281b2
Branch: refs/heads/v1.5.4-release2
Commit: fc7281b21df6bf48b10999bb10e270207540bb18
Parents: 9777878
Author: shaofengshi <sh...@apache.org>
Authored: Thu Sep 8 07:58:56 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Sep 8 07:58:56 2016 +0800
----------------------------------------------------------------------
.../model_desc/test_kylin_inner_join_model_desc.json | 1 -
.../model_desc/test_kylin_inner_join_view_model_desc.json | 9 ---------
.../model_desc/test_kylin_left_join_model_desc.json | 8 +++++++-
.../model_desc/test_kylin_left_join_view_model_desc.json | 9 ---------
4 files changed, 7 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index f5333b0..cb43d6f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
"name" : "test_kylin_inner_join_model_desc",
"lookups" : [ {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
index 54e9ffb..acdcab8 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
@@ -1,17 +1,8 @@
{
-
"uuid": "8b184ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_inner_join_view_model_desc",
"lookups": [
{
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "inner",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
- }
- },
- {
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "inner",
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index bedb167..735ca05 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid": "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_left_join_model_desc",
"lookups": [
@@ -82,6 +81,13 @@
"seller_type_cd",
"seller_type_desc"
]
+ },
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "columns": [
+ "cal_dt",
+ "week_beg_dt"
+ ]
}
],
"metrics": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc7281b2/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
index 7c009a1..a4af260 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
@@ -1,17 +1,8 @@
{
-
"uuid": "9c184ee2-1ccb-4b07-a38e-4c298563e0f7",
"name": "test_kylin_left_join_view_model_desc",
"lookups": [
{
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
- }
- },
- {
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "left",
[02/43] kylin git commit: KYLIN-1698 fix UT
Posted by sh...@apache.org.
KYLIN-1698 fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/111e984f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/111e984f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/111e984f
Branch: refs/heads/v1.5.4-release2
Commit: 111e984f9c3a47a8a90cc622ef368cf68b0f84fe
Parents: a20d709
Author: Li Yang <li...@apache.org>
Authored: Fri Sep 2 16:11:14 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Sep 2 16:11:14 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/metadata/model/PartitionDesc.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/111e984f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 8d3979e..598e6d0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -92,6 +92,9 @@ public class PartitionDesc {
}
public boolean partitionColumnIsTimeMillis() {
+ if (partitionDateColumnRef == null)
+ return false;
+
DataType type = partitionDateColumnRef.getType();
return type.isBigInt();
}
[10/43] kylin git commit: KYLIN-1965: bug fix
Posted by sh...@apache.org.
KYLIN-1965: bug fix
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08f7ddab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08f7ddab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08f7ddab
Branch: refs/heads/v1.5.4-release2
Commit: 08f7ddab1c63db4bcd4d8d461a8a21da0586512d
Parents: 7bc420b
Author: kangkaisen <ka...@live.com>
Authored: Mon Sep 5 14:11:37 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Sep 6 11:04:30 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/08f7ddab/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 794d2b2..2f191f9 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -206,7 +206,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
names.push(measures[i].name);
}
var index = names.indexOf(newMeasure.name);
- return index > -1;
+ return (index > -1 && index != $scope.updateMeasureStatus.editIndex);
}
$scope.nextParameterInit = function(){
[34/43] kylin git commit: remove unnecessary raw measure cond
Posted by sh...@apache.org.
remove unnecessary raw measure cond
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a05f1114
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a05f1114
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a05f1114
Branch: refs/heads/v1.5.4-release2
Commit: a05f11146c9edd4ba18618f38fd13c9b7e4806ce
Parents: 942406b
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 23:45:17 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 23:45:17 2016 +0800
----------------------------------------------------------------------
.../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 3 ---
.../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 3 ---
2 files changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 7de2ae2..d185175 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -234,9 +234,6 @@
"item_count_sum",
"SITE_EXTENDED_1",
"SITE_EXTENDED_2",
- "CAL_DT_RAW",
- "LSTG_FORMAT_NAME_RAW",
- "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a05f1114/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 4270aab..2aea1a8 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -266,9 +266,6 @@
"gmv_max",
"trans_cnt",
"item_count_sum",
- "CAL_DT_RAW",
- "LSTG_FORMAT_NAME_RAW",
- "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
[09/43] kylin git commit: KYLIN-Storage-Engine-kylin
Posted by sh...@apache.org.
KYLIN-Storage-Engine-kylin
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7bc420b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7bc420b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7bc420b2
Branch: refs/heads/v1.5.4-release2
Commit: 7bc420b211f506c789a15d214b6730591a583ea0
Parents: a9c8f40
Author: zx chen <34...@qq.com>
Authored: Fri Sep 2 18:19:54 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Sep 5 12:06:09 2016 +0800
----------------------------------------------------------------------
webapp/app/js/model/cubeDescModel.js | 6 +++---
webapp/app/js/services/kylinProperties.js | 16 ++++++++++++++++
2 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7bc420b2/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index c981249..95a0b13 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-KylinApp.service('CubeDescModel', function () {
+KylinApp.service('CubeDescModel', function (kylinConfig) {
this.cubeMetaFrame = {};
@@ -57,8 +57,8 @@ KylinApp.service('CubeDescModel', function () {
"retention_range": "0",
"status_need_notify":['ERROR', 'DISCARDED', 'SUCCEED'],
"auto_merge_time_ranges": [604800000, 2419200000],
- "engine_type": 2,
- "storage_type":2,
+ "engine_type": kylinConfig.getCubeEng(),
+ "storage_type":kylinConfig.getStorageEng(),
"override_kylin_properties":{}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/7bc420b2/webapp/app/js/services/kylinProperties.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js
index 68e8766..7110b8c 100644
--- a/webapp/app/js/services/kylinProperties.js
+++ b/webapp/app/js/services/kylinProperties.js
@@ -71,6 +71,22 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
}
return this.hiveLimit;
}
+
+ this.getStorageEng = function () {
+ this.StorageEng = this.getProperty("kylin.default.storage.engine").trim();
+ if (!this.StorageEng) {
+ return 2;
+ }
+ return this.StorageEng;
+ }
+
+ this.getCubeEng = function () {
+ this.CubeEng = this.getProperty("kylin.default.cube.engine").trim();
+ if (!this.CubeEng) {
+ return 2;
+ }
+ return this.CubeEng;
+ }
//fill config info for Config from backend
this.initWebConfigInfo = function () {
[25/43] kylin git commit: minor, add diagnosis to smoke test
Posted by sh...@apache.org.
minor, add diagnosis to smoke test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf261148
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf261148
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf261148
Branch: refs/heads/v1.5.4-release2
Commit: bf26114866d9b4f8dcb5e13c48acdbf6c8e308e9
Parents: be7751b
Author: lidongsjtu <li...@apache.org>
Authored: Fri Sep 9 12:37:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Sep 9 12:37:05 2016 +0800
----------------------------------------------------------------------
build/smoke-test/smoke-test.sh | 1 +
build/smoke-test/testDiag.py | 44 +++++++++++++++++++++++++++++++++++++
2 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf261148/build/smoke-test/smoke-test.sh
----------------------------------------------------------------------
diff --git a/build/smoke-test/smoke-test.sh b/build/smoke-test/smoke-test.sh
index 06d930a..c21bd6d 100755
--- a/build/smoke-test/smoke-test.sh
+++ b/build/smoke-test/smoke-test.sh
@@ -67,6 +67,7 @@ sleep 3m
cd $dir/smoke-test
python testBuildCube.py || { exit 1; }
python testQuery.py || { exit 1; }
+python testDiag.py || { exit 1; }
cd -
# Tear down stage
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf261148/build/smoke-test/testDiag.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
new file mode 100644
index 0000000..cc932da
--- /dev/null
+++ b/build/smoke-test/testDiag.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is python unittest used in smoke-test.sh, aim to testing query via rest APIs.
+
+import unittest
+import requests
+
+class testDiag(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def testDiag(self):
+ url = "http://sandbox:7070/kylin/api/diag/project/learn_kylin/download"
+ headers = {
+ 'content-type': "application/json",
+ 'authorization': "Basic QURNSU46S1lMSU4=",
+ 'cache-control': "no-cache"
+ }
+
+ response = requests.get(url, headers = headers)
+ self.assertEqual(response.status_code, 200, 'Diagnosis failed.')
+
+
+if __name__ == '__main__':
+ print 'Test Diagnogis for Kylin sample.'
+ unittest.main()
[21/43] kylin git commit: minor, add one more test case
Posted by sh...@apache.org.
minor, add one more test case
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01d033f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01d033f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01d033f7
Branch: refs/heads/v1.5.4-release2
Commit: 01d033f76a058ca0cf7ef81728b7de6204115167
Parents: ddeb745
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Sep 8 11:18:52 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 8 11:18:52 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeCapabilityChecker.java | 7 ++++++-
kylin-it/src/test/resources/query/sql_like/query18.sql | 13 +++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/01d033f7/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 79d1e3b..e8c96b4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -31,6 +31,7 @@ import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -146,7 +147,11 @@ public class CubeCapabilityChecker {
}
// calcite can do aggregation from columns on-the-fly
- List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
+ ParameterDesc parameterDesc = functionDesc.getParameter();
+ if (parameterDesc == null) {
+ continue;
+ }
+ List<TblColRef> neededCols = parameterDesc.getColRefs();
if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
it.remove();
http://git-wip-us.apache.org/repos/asf/kylin/blob/01d033f7/kylin-it/src/test/resources/query/sql_like/query18.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query18.sql b/kylin-it/src/test/resources/query/sql_like/query18.sql
new file mode 100644
index 0000000..8ef6ad4
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query18.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where upper(USER_DEFINED_FIELD3) like '%VID%'
[41/43] kylin git commit: KYLIN-1983 add license header
Posted by sh...@apache.org.
KYLIN-1983 add license header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b941f115
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b941f115
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b941f115
Branch: refs/heads/v1.5.4-release2
Commit: b941f115d5dae06397605c73cb630fbc31c9a5ab
Parents: 5e95abd
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 09:29:45 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
.../MultipleDictionaryValueEnumeratorTest.java | 18 ++++++++++++++++++
.../localmeta/kylin_account.properties | 17 +++++++++++++++++
2 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b941f115/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 6e0f88a..ed8a3c2 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.dict;
import org.apache.kylin.common.util.Bytes;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b941f115/examples/test_case_data/localmeta/kylin_account.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_account.properties b/examples/test_case_data/localmeta/kylin_account.properties
index 67bbb16..ac34172 100644
--- a/examples/test_case_data/localmeta/kylin_account.properties
+++ b/examples/test_case_data/localmeta/kylin_account.properties
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
### JOB ###
# Only necessary when kylin.job.run.as.remote.cmd=true
[20/43] kylin git commit: more test queries and minor refactor
Posted by sh...@apache.org.
more test queries and minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ddeb7452
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ddeb7452
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ddeb7452
Branch: refs/heads/v1.5.4-release2
Commit: ddeb7452a63279311b74fcfb5f7ed0252e3d54fd
Parents: fc7281b
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Sep 6 16:09:59 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 8 10:15:08 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 63 +-------
.../filter/BuiltInFunctionTupleFilter.java | 12 +-
.../filter/EvaluatableFunctionTupleFilter.java | 151 +++++++++++++++++++
.../filter/EvaluatableLikeFunction.java | 114 --------------
.../kylin/metadata/filter/TupleFilter.java | 2 +-
.../metadata/filter/TupleFilterSerializer.java | 4 +-
.../metadata/filter/function/BuiltInMethod.java | 9 ++
.../test_kylin_inner_join_model_desc.json | 103 ++++++++-----
.../test_kylin_inner_join_view_model_desc.json | 24 ++-
.../test_kylin_left_join_model_desc.json | 31 ++--
.../test_kylin_left_join_view_model_desc.json | 24 ++-
.../apache/kylin/query/ITKylinQueryTest.java | 25 +--
.../src/test/resources/query/sql/query45.sql | 24 ---
.../src/test/resources/query/sql/query46.sql | 19 ---
.../src/test/resources/query/sql/query47.sql | 19 ---
.../src/test/resources/query/sql/query48.sql | 19 ---
.../src/test/resources/query/sql/query55.sql | 19 ---
.../src/test/resources/query/sql/query83.sql | 33 ++++
.../src/test/resources/query/sql/query84.sql | 33 ++++
.../src/test/resources/query/sql/query92.sql | 6 +-
.../src/test/resources/query/sql/query93.sql | 6 +-
.../src/test/resources/query/sql/query94.sql | 6 +-
.../src/test/resources/query/sql/query95.sql | 6 +-
.../test/resources/query/sql_like/query04.sql | 22 +++
.../test/resources/query/sql_like/query10.sql | 13 ++
.../test/resources/query/sql_like/query15.sql | 13 ++
.../test/resources/query/sql_like/query16.sql | 13 ++
.../test/resources/query/sql_like/query17.sql | 13 ++
.../test/resources/query/sql_lookup/query45.sql | 24 +++
.../test/resources/query/sql_lookup/query46.sql | 19 +++
.../test/resources/query/sql_lookup/query47.sql | 19 +++
.../test/resources/query/sql_lookup/query48.sql | 19 +++
.../test/resources/query/sql_lookup/query55.sql | 19 +++
.../test/resources/query/sql_raw/query21.sql | 24 +++
.../test/resources/query/sql_raw/query22.sql | 24 +++
.../test/resources/query/sql_raw/query23.sql | 24 +++
.../test/resources/query/sql_raw/query24.sql | 24 +++
.../test/resources/query/sql_raw/query25.sql | 24 +++
.../query/sql_raw/query26.sql.disabled | 24 +++
.../kylin/query/relnode/OLAPAggregateRel.java | 15 +-
.../apache/kylin/query/relnode/OLAPContext.java | 7 +
.../apache/kylin/query/relnode/OLAPJoinRel.java | 40 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 6 +-
.../apache/kylin/query/routing/Candidate.java | 5 +-
44 files changed, 753 insertions(+), 390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 7406e81..8d310a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -18,7 +18,6 @@
package org.apache.kylin.gridtable;
-import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
@@ -26,7 +25,6 @@ import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -108,11 +106,11 @@ public class GTUtil {
};
}
- private static class GTConvertDecorator implements TupleFilterSerializer.Decorator {
- private final Set<TblColRef> unevaluatableColumnCollector;
- private final List<TblColRef> colMapping;
- private final GTInfo info;
- private final boolean encodeConstants;
+ protected static class GTConvertDecorator implements TupleFilterSerializer.Decorator {
+ protected final Set<TblColRef> unevaluatableColumnCollector;
+ protected final List<TblColRef> colMapping;
+ protected final GTInfo info;
+ protected final boolean encodeConstants;
public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, List<TblColRef> colMapping, GTInfo info, boolean encodeConstants) {
this.unevaluatableColumnCollector = unevaluatableColumnCollector;
@@ -152,19 +150,12 @@ public class GTUtil {
if (encodeConstants && filter instanceof CompareTupleFilter) {
return encodeConstants((CompareTupleFilter) filter);
}
- if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
- if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
- return encodeConstants((BuiltInFunctionTupleFilter) filter);
- } else {
- throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
- }
- }
return filter;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
+ protected TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
// extract ColumnFilter & ConstantFilter
TblColRef externalCol = oldCompareFilter.getColumn();
@@ -258,49 +249,9 @@ public class GTUtil {
return result;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
- // extract ColumnFilter & ConstantFilter
- TblColRef externalCol = funcFilter.getColumn();
-
- if (externalCol == null) {
- return funcFilter;
- }
-
- Collection constValues = funcFilter.getConstantTupleFilter().getValues();
- if (constValues == null || constValues.isEmpty()) {
- return funcFilter;
- }
-
- BuiltInFunctionTupleFilter newFuncFilter;
- try {
- newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
- } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
- throw new RuntimeException(e);
- }
- newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
-
- int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
-
- ByteArray code;
-
- // translate constant into code
- Set newValues = Sets.newHashSet();
- for (Object value : constValues) {
- code = translate(col, value, 0);
- if (code == null) {
- throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
- }
- newValues.add(code);
- }
- newFuncFilter.addChild(new ConstantTupleFilter(newValues));
-
- return newFuncFilter;
- }
-
transient ByteBuffer buf;
- private ByteArray translate(int col, Object value, int roundingFlag) {
+ protected ByteArray translate(int col, Object value, int roundingFlag) {
try {
buf.clear();
info.codeSystem.encodeColumnValue(col, value, roundingFlag, buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 40afb18..734b374 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -43,6 +43,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
protected ConstantTupleFilter constantTupleFilter;
protected int colPosition;
+ protected int constantPosition;
protected Method method;
protected List<Serializable> methodParams;
protected boolean isValidFunc = false;
@@ -69,6 +70,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
return constantTupleFilter;
}
+ public TupleFilter getColumnContainerFilter() {
+ return columnContainerFilter;
+ }
+
public TblColRef getColumn() {
if (columnContainerFilter == null)
return null;
@@ -81,10 +86,6 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
}
- public boolean hasNested() {
- return (columnContainerFilter != null && columnContainerFilter instanceof BuiltInFunctionTupleFilter);
- }
-
public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
if (columnContainerFilter instanceof ColumnTupleFilter)
methodParams.set(colPosition, (Serializable) input);
@@ -107,6 +108,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
this.constantTupleFilter = (ConstantTupleFilter) child;
Serializable constVal = (Serializable) child.getValues().iterator().next();
try {
+ constantPosition = methodParams.size();
Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
if (!Primitives.isWrapperType(clazz))
methodParams.add(constVal);
@@ -131,7 +133,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
}
@Override
- public Collection<String> getValues() {
+ public Collection<?> getValues() {
throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
new file mode 100644
index 0000000..ff24172
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.StringSerializer;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
+import com.google.common.collect.Lists;
+
+public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter {
+
+ private boolean constantsInitted = false;
+
+ //about non-like
+ private List<Object> values;
+ private Object tupleValue;
+
+ public EvaluatableFunctionTupleFilter(String name) {
+ super(name, FilterOperatorEnum.EVAL_FUNC);
+ values = Lists.newArrayListWithCapacity(1);
+ values.add(null);
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
+
+ // extract tuple value
+ Object tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ if (!isConstant(filter)) {
+ filter.evaluate(tuple, cs);
+ tupleValue = filter.getValues().iterator().next();
+ }
+ }
+
+ TblColRef tblColRef = this.getColumn();
+ DataType strDataType = DataType.getType("string");
+ if (tblColRef.getType() != strDataType) {
+ throw new IllegalStateException("Only String type is allow in BuiltInFunction");
+ }
+ ByteArray valueByteArray = (ByteArray) tupleValue;
+ StringSerializer serializer = new StringSerializer(strDataType);
+ String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
+
+ try {
+ if (isLikeFunction()) {
+ return (Boolean) invokeFunction(value);
+ } else {
+ this.tupleValue = invokeFunction(value);
+ //convert back to ByteArray format because the outer EvaluatableFunctionTupleFilter assumes input as ByteArray
+ ByteBuffer buffer = ByteBuffer.allocate(valueByteArray.length() * 2);
+ serializer.serialize((String) this.tupleValue, buffer);
+ this.tupleValue = new ByteArray(buffer.array(), 0, buffer.position());
+
+ return true;
+ }
+ } catch (InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Collection<?> getValues() {
+ this.values.set(0, tupleValue);
+ return values;
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ if (!isValid()) {
+ throw new IllegalStateException("must be valid");
+ }
+ BytesUtil.writeUTFString(name, buffer);
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ private boolean isConstant(TupleFilter filter) {
+ return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+ }
+
+ @Override
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (isLikeFunction())
+ initConstants();
+ return super.invokeFunction(input);
+ }
+
+ private void initConstants() {
+ if (constantsInitted) {
+ return;
+ }
+ //will replace the ByteArray pattern to String type
+ ByteArray byteArray = (ByteArray) methodParams.get(constantPosition);
+ StringSerializer s = new StringSerializer(DataType.getType("string"));
+ String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
+ //TODO
+ //pattern = pattern.toLowerCase();//to remove upper case
+ methodParams.set(constantPosition, pattern);
+ constantsInitted = true;
+ }
+
+ //even for "tolower(s)/toupper(s)/substring(like) like pattern", the like pattern can be used for index searching
+ public String getLikePattern() {
+ if (!isLikeFunction()) {
+ return null;
+ }
+
+ initConstants();
+ return (String) methodParams.get(1);
+ }
+
+ public boolean isLikeFunction() {
+ return "like".equalsIgnoreCase(this.getName());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
deleted file mode 100644
index 28e544b..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-/**
- * typically like will be handled by BuiltInFunctionTupleFilter rather than this
- */
-public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
-
- private boolean patternExtracted;
-
- public EvaluatableLikeFunction(String name) {
- super(name, FilterOperatorEnum.LIKE);
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
- // extract tuple value
- Object tupleValue = null;
- for (TupleFilter filter : this.children) {
- if (!isConstant(filter)) {
- filter.evaluate(tuple, cs);
- tupleValue = filter.getValues().iterator().next();
- }
- }
-
- // consider null case
- if (cs.isNull(tupleValue)) {
- return false;
- }
-
- ByteArray valueByteArray = (ByteArray) tupleValue;
- StringSerializer serializer = new StringSerializer(DataType.getType("string"));
- String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
- try {
- return (Boolean) invokeFunction(value);
- } catch (InvocationTargetException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- if (!isValid()) {
- throw new IllegalStateException("must be valid");
- }
- if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1) == null) {
- throw new IllegalArgumentException("bad methodParams: " + methodParams);
- }
- BytesUtil.writeUTFString(name, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- private boolean isConstant(TupleFilter filter) {
- return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
- }
-
- @Override
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (!patternExtracted) {
- this.getLikePattern();
- }
- return super.invokeFunction(input);
- }
-
- //will replace the ByteArray pattern to String type
- public String getLikePattern() {
- if (!patternExtracted) {
- ByteArray byteArray = (ByteArray) methodParams.get(1);
- StringSerializer s = new StringSerializer(DataType.getType("string"));
- String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
- methodParams.set(1, pattern);
- patternExtracted = true;
- }
- return (String) methodParams.get(1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 2fb4e1f..900cc35 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -38,7 +38,7 @@ import com.google.common.collect.Maps;
public abstract class TupleFilter {
public enum FilterOperatorEnum {
- EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), LIKE(37);
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37);
private final int value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 0aca9a1..04984f2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -188,8 +188,8 @@ public class TupleFilterSerializer {
case FUNCTION:
filter = new BuiltInFunctionTupleFilter(null);
break;
- case LIKE:
- filter = new EvaluatableLikeFunction(null);
+ case EVAL_FUNC:
+ filter = new EvaluatableFunctionTupleFilter(null);
break;
case MASSIN:
filter = new MassInTupleFilter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index 7b241cc..2f28fae 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -47,6 +47,9 @@ public enum BuiltInMethod {
/** SQL {@code LIKE} function. */
public static boolean like(String s, String pattern) {
+ if (s == null)
+ return false;
+
final String regex = Like.sqlToRegexLike(pattern, null);
return Pattern.matches(regex, s);
}
@@ -95,16 +98,22 @@ public enum BuiltInMethod {
/** SQL SUBSTRING(string FROM ... FOR ...) function. */
public static String substring(String s, int from, int for_) {
+ if (s == null)
+ return null;
return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
}
/** SQL UPPER(string) function. */
public static String upper(String s) {
+ if (s == null)
+ return null;
return s.toUpperCase();
}
/** SQL LOWER(string) function. */
public static String lower(String s) {
+ if (s == null)
+ return null;
return s.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index cb43d6f..6e444e5 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -1,35 +1,58 @@
{
- "uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
- "name" : "test_kylin_inner_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
+ "uuid": "ff527b94-f860-44c3-8452-93b17774c647",
+ "name": "test_kylin_inner_join_model_desc",
+ "lookups": [
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "LEAF_CATEG_ID",
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LEAF_CATEG_ID",
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "inner",
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
+ }
}
- } ],
+ ],
"dimensions": [
{
"table": "default.test_kylin_fact",
@@ -81,16 +104,16 @@
}
],
"metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID"
+ "PRICE",
+ "ITEM_COUNT",
+ "SELLER_ID"
],
- "last_modified" : 1422435345352,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
+ "last_modified": 1422435345352,
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
index acdcab8..621eccc 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "inner",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "inner",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "inner",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index 735ca05..dab99f9 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "left",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "left",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
@@ -45,11 +57,12 @@
{
"table": "default.test_kylin_fact",
"columns": [
- "TRANS_ID",
- "CAL_DT",
"lstg_format_name",
"LSTG_SITE_ID",
"SLR_SEGMENT_CD",
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
"SELLER_ID"
]
},
@@ -83,7 +96,7 @@
]
},
{
- "table": "EDW.TEST_CAL_DT",
+ "table": "edw.test_cal_dt",
"columns": [
"cal_dt",
"week_beg_dt"
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
index a4af260..819b8b0 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
@@ -6,8 +6,12 @@
"table": "EDW.V_TEST_CAL_DT",
"join": {
"type": "left",
- "primary_key": ["CAL_DT"],
- "foreign_key": ["CAL_DT"]
+ "primary_key": [
+ "CAL_DT"
+ ],
+ "foreign_key": [
+ "CAL_DT"
+ ]
}
},
{
@@ -28,16 +32,24 @@
"table": "EDW.TEST_SITES",
"join": {
"type": "left",
- "primary_key": ["SITE_ID"],
- "foreign_key": ["LSTG_SITE_ID"]
+ "primary_key": [
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LSTG_SITE_ID"
+ ]
}
},
{
"table": "EDW.TEST_SELLER_TYPE_DIM",
"join": {
"type": "left",
- "primary_key": ["SELLER_TYPE_CD"],
- "foreign_key": ["SLR_SEGMENT_CD"]
+ "primary_key": [
+ "SELLER_TYPE_CD"
+ ],
+ "foreign_key": [
+ "SLR_SEGMENT_CD"
+ ]
}
}
],
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4657a5c..375b198 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -55,12 +55,13 @@ public class ITKylinQueryTest extends KylinTestBase {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
Candidate.setPriorities(priorities);
- joinType = "inner";
+ joinType = "left";
setupAll();
-
+
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
@@ -192,18 +193,25 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
- public void testDimDistinctCountQuery() throws Exception {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
+ public void testTopNQuery() throws Exception {
+ if ("left".equalsIgnoreCase(joinType)) {
+ this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true);
+ }
}
@Test
public void testPreciselyDistinctCountQuery() throws Exception {
if ("left".equalsIgnoreCase(joinType)) {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/temp", null, true);
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_precisely", null, true);
}
}
@Test
+ public void testDimDistinctCountQuery() throws Exception {
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
+ }
+
+ @Test
public void testStreamingTableQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_streaming", null, true);
}
@@ -276,13 +284,6 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
- public void testTopNQuery() throws Exception {
- if ("left".equalsIgnoreCase(joinType)) {
- this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true);
- }
- }
-
- @Test
public void testRawQuery() throws Exception {
this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_raw", null, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query45.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query45.sql b/kylin-it/src/test/resources/query/sql/query45.sql
deleted file mode 100644
index ea964ae..0000000
--- a/kylin-it/src/test/resources/query/sql/query45.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_cal_dt
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query46.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query46.sql b/kylin-it/src/test/resources/query/sql/query46.sql
deleted file mode 100644
index 3bfe9d9..0000000
--- a/kylin-it/src/test/resources/query/sql/query46.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from test_category_groupings
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query47.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query47.sql b/kylin-it/src/test/resources/query/sql/query47.sql
deleted file mode 100644
index cbd2c6d..0000000
--- a/kylin-it/src/test/resources/query/sql/query47.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_seller_type_dim
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query48.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query48.sql b/kylin-it/src/test/resources/query/sql/query48.sql
deleted file mode 100644
index 54ddb31..0000000
--- a/kylin-it/src/test/resources/query/sql/query48.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as CNT from edw.test_sites
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query55.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query55.sql b/kylin-it/src/test/resources/query/sql/query55.sql
deleted file mode 100644
index 346a7d7..0000000
--- a/kylin-it/src/test/resources/query/sql/query55.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select count(*) as c from edw.test_cal_dt as test_cal_dt where extract(YEAR from test_cal_dt.cal_dt) = 2012
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query83.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query83.sql b/kylin-it/src/test/resources/query/sql/query83.sql
new file mode 100644
index 0000000..1fbbd33
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query83.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(1) as TRANS_CNT
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and
+ test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ on test_kylin_fact.lstg_site_id = test_sites.site_id
+ left JOIN edw.test_seller_type_dim as test_seller_type_dim
+ on test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd
+ where test_kylin_fact.lstg_format_name='FP-GTC'
+ and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ and test_kylin_fact.cal_dt between DATE '2013-06-01' and DATE '2013-09-01'
+ group by test_cal_dt.week_beg_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query84.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query84.sql b/kylin-it/src/test/resources/query/sql/query84.sql
new file mode 100644
index 0000000..2560465
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query84.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+ select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(1) as TRANS_CNT
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ on test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id and
+ test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ on test_kylin_fact.lstg_site_id = test_sites.site_id
+ left JOIN edw.test_seller_type_dim as test_seller_type_dim
+ on test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd
+ where test_kylin_fact.lstg_format_name='FP-GTC'
+ and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ and test_cal_dt.cal_dt between DATE '2013-06-01' and DATE '2013-09-01'
+ group by test_cal_dt.week_beg_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query92.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query92.sql b/kylin-it/src/test/resources/query/sql/query92.sql
index e551a45..ebc07fe 100644
--- a/kylin-it/src/test/resources/query/sql/query92.sql
+++ b/kylin-it/src/test/resources/query/sql/query92.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('', 'a')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query93.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query93.sql b/kylin-it/src/test/resources/query/sql/query93.sql
index cc6dca5..085e0e6 100644
--- a/kylin-it/src/test/resources/query/sql/query93.sql
+++ b/kylin-it/src/test/resources/query/sql/query93.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name is not null
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query94.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query94.sql b/kylin-it/src/test/resources/query/sql/query94.sql
index c7899fd..b39d163 100644
--- a/kylin-it/src/test/resources/query/sql/query94.sql
+++ b/kylin-it/src/test/resources/query/sql/query94.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('Unknown')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql/query95.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query95.sql b/kylin-it/src/test/resources/query/sql/query95.sql
index 578b93f..fd94dcc 100644
--- a/kylin-it/src/test/resources/query/sql/query95.sql
+++ b/kylin-it/src/test/resources/query/sql/query95.sql
@@ -19,11 +19,11 @@
select meta_categ_name, count(1) as cnt, sum(price) as GMV
from test_kylin_fact
- left JOIN edw.test_cal_dt as test_cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- left JOIN test_category_groupings
+ inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- left JOIN edw.test_sites as test_sites
+ inner JOIN edw.test_sites as test_sites
ON test_kylin_fact.lstg_site_id = test_sites.site_id
where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines')
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query04.sql b/kylin-it/src/test/resources/query/sql_like/query04.sql
new file mode 100644
index 0000000..faf5ca3
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query04.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='auction' and substring(lstg_format_name,1,3) in ('Auc') and upper(lstg_format_name) > 'AAAA' and
+upper(lstg_format_name) like '%UC%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='Auctiona'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query10.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query10.sql b/kylin-it/src/test/resources/query/sql_like/query10.sql
new file mode 100644
index 0000000..21632f4
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query10.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where USER_DEFINED_FIELD3 like '%Video Game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query15.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query15.sql b/kylin-it/src/test/resources/query/sql_like/query15.sql
new file mode 100644
index 0000000..85418c7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query15.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lower(USER_DEFINED_FIELD3) like '%Video Game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query16.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query16.sql b/kylin-it/src/test/resources/query/sql_like/query16.sql
new file mode 100644
index 0000000..2a484ef
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query16.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lower(USER_DEFINED_FIELD3) like '%video game%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_like/query17.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query17.sql b/kylin-it/src/test/resources/query/sql_like/query17.sql
new file mode 100644
index 0000000..c6dd1ea
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query17.sql
@@ -0,0 +1,13 @@
+
+select USER_DEFINED_FIELD3 as abc
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where upper(USER_DEFINED_FIELD3) like '%VIDEO GAME%'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query45.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query45.sql b/kylin-it/src/test/resources/query/sql_lookup/query45.sql
new file mode 100644
index 0000000..ea964ae
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query45.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_cal_dt
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query46.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query46.sql b/kylin-it/src/test/resources/query/sql_lookup/query46.sql
new file mode 100644
index 0000000..3bfe9d9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query46.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from test_category_groupings
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query47.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query47.sql b/kylin-it/src/test/resources/query/sql_lookup/query47.sql
new file mode 100644
index 0000000..cbd2c6d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query47.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_seller_type_dim
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query48.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query48.sql b/kylin-it/src/test/resources/query/sql_lookup/query48.sql
new file mode 100644
index 0000000..54ddb31
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query48.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_sites
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_lookup/query55.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query55.sql b/kylin-it/src/test/resources/query/sql_lookup/query55.sql
new file mode 100644
index 0000000..346a7d7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query55.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as c from edw.test_cal_dt as test_cal_dt where extract(YEAR from test_cal_dt.cal_dt) = 2012
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query21.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query21.sql b/kylin-it/src/test/resources/query/sql_raw/query21.sql
new file mode 100644
index 0000000..4905e0f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query21.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query22.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query22.sql b/kylin-it/src/test/resources/query/sql_raw/query22.sql
new file mode 100644
index 0000000..d603a25
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query22.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query23.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query23.sql b/kylin-it/src/test/resources/query/sql_raw/query23.sql
new file mode 100644
index 0000000..89d61db
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query23.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query24.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query24.sql b/kylin-it/src/test/resources/query/sql_raw/query24.sql
new file mode 100644
index 0000000..46c7329
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query24.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT,LSTG_FORMAT_NAME from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id where LSTG_FORMAT_NAME = 'ABIN'
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query25.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query25.sql b/kylin-it/src/test/resources/query/sql_raw/query25.sql
new file mode 100644
index 0000000..175124f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query25.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.CAL_DT from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled b/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
new file mode 100644
index 0000000..cab92d0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_raw/query26.sql.disabled
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select LSTG_FORMAT_NAME,LSTG_SITE_ID,SLR_SEGMENT_CD,test_kylin_fact.CAL_DT,test_category_groupings.LEAF_CATEG_ID,PRICE from test_kylin_fact inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index ba74c74..f55c86f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.Sets;
import org.apache.calcite.adapter.enumerable.EnumerableAggregate;
import org.apache.calcite.adapter.enumerable.EnumerableConvention;
import org.apache.calcite.adapter.enumerable.EnumerableRel;
@@ -67,6 +66,7 @@ import org.apache.kylin.query.schema.OLAPTable;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
*/
@@ -133,8 +133,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
if (getGroupType() == Group.SIMPLE) {
cost = super.computeSelfCost(planner, mq).multiplyBy(.05);
} else {
- cost = super.computeSelfCost(planner, mq).multiplyBy(.05).plus(planner.getCost(getInput(), mq))
- .multiplyBy(groupSets.size() * 1.5);
+ cost = super.computeSelfCost(planner, mq).multiplyBy(.05).plus(planner.getCost(getInput(), mq)).multiplyBy(groupSets.size() * 1.5);
}
return cost;
}
@@ -174,7 +173,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// Add group column indicators
if (indicator) {
final Set<String> containedNames = Sets.newHashSet();
- for (TblColRef groupCol: groups) {
+ for (TblColRef groupCol : groups) {
String base = "i$" + groupCol.getName();
String name = base;
int i = 0;
@@ -356,6 +355,14 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
private AggregateCall rewriteAggregateCall(AggregateCall aggCall, FunctionDesc func) {
+
+ //if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
+ // which do not exist at all
+ if (!this.context.hasPrecalculatedFields() && func.needRewriteField()) {
+ logger.info(func + "skip rewriteAggregateCall because no pre-aggregated field available");
+ return aggCall;
+ }
+
// rebuild parameters
List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
if (func.needRewriteField()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index c2e1b88..41a3b4d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
@@ -38,6 +39,7 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import com.google.common.collect.Lists;
@@ -146,6 +148,10 @@ public class OLAPContext {
return sqlDigest;
}
+ public boolean hasPrecalculatedFields() {
+ return realization instanceof CubeInstance || realization instanceof HybridInstance;
+ }
+
public void resetSQLDigest() {
this.sqlDigest = null;
}
@@ -167,6 +173,7 @@ public class OLAPContext {
sortOrders.add(order);
}
}
+
public interface IAccessController {
/*
* @return {TupleFilter} if the filter condition exists
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6dbb81a..2a143fb 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -48,9 +48,9 @@ import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
@@ -166,6 +166,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) {
TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue());
this.context.groupByColumns.add(fromCol);
+ this.context.allColumns.add(fromCol);
}
joinCol.clear();
}
@@ -295,26 +296,29 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
this.rowType = this.deriveRowType();
if (this.isTopJoin && RewriteImplementor.needRewrite(this.context)) {
- // find missed rewrite fields
- int paramIndex = this.rowType.getFieldList().size();
- List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
- for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
- String fieldName = rewriteField.getKey();
- if (this.rowType.getField(fieldName, true, false) == null) {
- RelDataType fieldType = rewriteField.getValue();
- RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, paramIndex++, fieldType);
- newFieldList.add(newField);
+ if (this.context.hasPrecalculatedFields()) {
+
+ // find missed rewrite fields
+ int paramIndex = this.rowType.getFieldList().size();
+ List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
+ for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
+ String fieldName = rewriteField.getKey();
+ if (this.rowType.getField(fieldName, true, false) == null) {
+ RelDataType fieldType = rewriteField.getValue();
+ RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, paramIndex++, fieldType);
+ newFieldList.add(newField);
+ }
}
- }
- // rebuild row type
- FieldInfoBuilder fieldInfo = getCluster().getTypeFactory().builder();
- fieldInfo.addAll(this.rowType.getFieldList());
- fieldInfo.addAll(newFieldList);
- this.rowType = getCluster().getTypeFactory().createStructType(fieldInfo);
+ // rebuild row type
+ FieldInfoBuilder fieldInfo = getCluster().getTypeFactory().builder();
+ fieldInfo.addAll(this.rowType.getFieldList());
+ fieldInfo.addAll(newFieldList);
+ this.rowType = getCluster().getTypeFactory().createStructType(fieldInfo);
- // rebuild columns
- this.columnRowType = this.buildColumnRowType();
+ // rebuild columns
+ this.columnRowType = this.buildColumnRowType();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 0a8a15f..db6ec2d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -37,9 +37,9 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -97,7 +97,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
boolean hasRexOver = RexOver.containsOver(getProjects(), null);
- return super.computeSelfCost(planner, mq).multiplyBy(.05).multiplyBy(getProjects().size() * (hasRexOver ? 50 : 1));
+ return super.computeSelfCost(planner, mq).multiplyBy(.05).multiplyBy(getProjects().size() * (hasRexOver ? 50 : 1));
}
@Override
@@ -244,7 +244,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
this.rewriting = true;
// project before join or is just after OLAPToEnumerableConverter
- if (!RewriteImplementor.needRewrite(this.context) || (this.hasJoin && !this.afterJoin) || this.afterAggregate) {
+ if (!RewriteImplementor.needRewrite(this.context) || (this.hasJoin && !this.afterJoin) || this.afterAggregate || !(this.context.hasPrecalculatedFields())) {
this.columnRowType = this.buildColumnRowType();
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ddeb7452/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index 9ea8961..bc17721 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.routing;
+import java.util.Collections;
import java.util.Map;
import org.apache.kylin.metadata.realization.CapabilityResult;
@@ -40,12 +41,12 @@ public class Candidate implements Comparable<Candidate> {
/** for test only */
public static void setPriorities(Map<RealizationType, Integer> priorities) {
- PRIORITIES = priorities;
+ PRIORITIES = Collections.unmodifiableMap(priorities);
}
/** for test only */
public static void restorePriorities() {
- PRIORITIES = DEFAULT_PRIORITIES;
+ PRIORITIES = Collections.unmodifiableMap(DEFAULT_PRIORITIES);
}
// ============================================================================
[37/43] kylin git commit: KYLIN-2004 Make the creating intermediate
hive table steps configurable
Posted by sh...@apache.org.
KYLIN-2004 Make the creating intermediate hive table steps configurable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/233a699f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/233a699f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/233a699f
Branch: refs/heads/v1.5.4-release2
Commit: 233a699f3b6f7a6c64ecf43fb80108b56db61f5f
Parents: d7cbf67
Author: shaofengshi <sh...@apache.org>
Authored: Fri Sep 9 19:04:10 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 17:59:46 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 48 ++++--
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/job/execution/AbstractExecutable.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 2 +-
.../kylin/metadata/model/DataModelDesc.java | 8 +-
...t_kylin_cube_without_slr_left_join_desc.json | 3 +-
.../kylin/rest/controller/CubeController.java | 2 +-
.../source/hive/CreateFlatHiveTableStep.java | 32 +++-
.../apache/kylin/source/hive/HiveMRInput.java | 169 ++++++++++++++++++-
10 files changed, 234 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ac9d48..de9051c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -805,7 +805,7 @@ abstract public class KylinConfigBase implements Serializable {
setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize));
}
- public boolean getTableJoinTypeCheck() {
- return Boolean.valueOf(this.getOptional("kylin.table.join.strong.check", "true"));
+ public String getCreateFlatHiveTableMethod() {
+ return getOptional("kylin.hive.create.flat.table.method", "1");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index b39265d..699d084 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -107,14 +107,14 @@ public class JoinedFlatTable {
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) {
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig, boolean redistribute) {
StringBuilder sql = new StringBuilder();
sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
+ sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc, redistribute) + ";").append("\n");
return sql.toString();
}
- public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
+ public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc, boolean redistribute) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT" + "\n");
String tableAlias;
@@ -129,7 +129,15 @@ public class JoinedFlatTable {
}
appendJoinStatement(flatDesc, sql, tableAliasMap);
appendWhereStatement(flatDesc, sql, tableAliasMap);
- appendDistributeStatement(flatDesc, sql, tableAliasMap);
+ if (redistribute == true) {
+ String redistributeCol = null;
+ TblColRef distDcol = flatDesc.getDistributedBy();
+ if (distDcol != null) {
+ String tblAlias = tableAliasMap.get(distDcol.getTable());
+ redistributeCol = tblAlias + "." + distDcol.getName();
+ }
+ appendDistributeStatement(sql, redistributeCol);
+ }
return sql.toString();
}
@@ -228,14 +236,11 @@ public class JoinedFlatTable {
return result;
}
- private static void appendDistributeStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
- TblColRef distDcol = flatDesc.getDistributedBy();
-
- if (distDcol != null) {
- String tblAlias = tableAliasMap.get(distDcol.getTable());
- sql.append(" DISTRIBUTE BY ").append(tblAlias).append(".").append(distDcol.getName());
+ private static void appendDistributeStatement(StringBuilder sql, String redistributeCol) {
+ if (redistributeCol != null) {
+ sql.append(" DISTRIBUTE BY ").append(redistributeCol).append(";\n");
} else {
- sql.append(" DISTRIBUTE BY RAND()");
+ sql.append(" DISTRIBUTE BY RAND()").append(";\n");
}
}
@@ -280,4 +285,25 @@ public class JoinedFlatTable {
return hiveDataType.toLowerCase();
}
+ public static String generateSelectRowCountStatement(IJoinedFlatTableDesc intermediateTableDesc, String outputDir) {
+ StringBuilder sql = new StringBuilder();
+ sql.append("set hive.exec.compress.output=false;\n");
+ sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + intermediateTableDesc.getTableName() + ";\n");
+ return sql.toString();
+ }
+
+ public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc intermediateTableDesc) {
+ final String tableName = intermediateTableDesc.getTableName();
+ StringBuilder sql = new StringBuilder();
+ sql.append("INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName);
+
+ String redistributeCol = null;
+ TblColRef distDcol = intermediateTableDesc.getDistributedBy();
+ if (distDcol != null) {
+ redistributeCol = colName(distDcol.getCanonicalName());
+ }
+ appendDistributeStatement(sql, redistributeCol);
+ return sql.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 6084e7b..893c034 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -56,5 +56,6 @@ public final class ExecutableConstants {
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
public static final String STEP_NAME_UPDATE_II_INFO = "Update Inverted Index Info";
+ public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table";
public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 4dedad1..09f9b54 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -49,7 +49,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
protected static final String START_TIME = "startTime";
protected static final String END_TIME = "endTime";
- private static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
+ protected static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
protected int retry = 0;
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
index 0faf22a..1fe47f8 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
@@ -77,7 +77,7 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
@Test
public void testGenerateInsertSql() throws IOException {
- String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()), true);
System.out.println(sqls);
int length = sqls.length();
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7f5edfe..d04830b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -314,13 +314,7 @@ public class DataModelDesc extends RootPersistentEntity {
}
for (int i = 0; i < fkCols.length; i++) {
if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) {
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- final String msg = "Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype();
- if (kylinConfig.getTableJoinTypeCheck() == true) {
- throw new IllegalStateException(msg);
- } else {
- logger.warn(msg);
- }
+ logger.warn("Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index ca1b35c..0470dc6 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -289,7 +289,8 @@
"engine_type" : 2,
"storage_type" : 2,
"override_kylin_properties": {
- "kylin.job.cubing.inmem.sampling.hll.precision": "16"
+ "kylin.job.cubing.inmem.sampling.hll.precision": "16",
+ "kylin.hive.create.flat.table.method": "2"
},
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 7081d02..5397df7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -152,7 +152,7 @@ public class CubeController extends BasicController {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
- String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc);
+ String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false);
GeneralResponse repsonse = new GeneralResponse();
repsonse.setProperty("sql", sql);
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index cd32f9c..bcb9a38 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -76,8 +76,11 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
private void createFlatHiveTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(getInitStatement());
- hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
- hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); //disable merge
+ boolean useRedistribute = getUseRedistribute();
+ if (useRedistribute == true) {
+ hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
+ hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); //disable merge
+ }
hiveCmdBuilder.addStatement(getCreateTableStatement());
final String cmd = hiveCmdBuilder.toString();
@@ -101,13 +104,20 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig config = getCubeSpecificConfig();
try {
- long rowCount = readRowCountFromFile();
- if (!config.isEmptySegmentAllowed() && rowCount == 0) {
- stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
- return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+
+ boolean useRedistribute = getUseRedistribute();
+
+ int numReducers = 0;
+ if (useRedistribute == true) {
+ long rowCount = readRowCountFromFile();
+ if (!config.isEmptySegmentAllowed() && rowCount == 0) {
+ stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+
+ numReducers = determineNumReducer(config, rowCount);
}
- int numReducers = determineNumReducer(config, rowCount);
createFlatHiveTable(config, numReducers);
return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
@@ -125,6 +135,14 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
return getParam("HiveInit");
}
+ public void setUseRedistribute(boolean useRedistribute) {
+ setParam("useRedistribute", String.valueOf(useRedistribute));
+ }
+
+ public boolean getUseRedistribute() {
+ return Boolean.valueOf(getParam("useRedistribute"));
+ }
+
public void setCreateTableStatement(String sql) {
setParam("HiveRedistributeData", sql);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/233a699f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e3d7879..3ea9af5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,8 +19,10 @@
package org.apache.kylin.source.hive;
import java.io.IOException;
+import java.io.InputStream;
import java.util.Set;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -28,6 +30,11 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BufferedLogger;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -110,16 +117,46 @@ public class HiveMRInput implements IMRInput {
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
- final String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()) + "/row_count";
+ final KylinConfig kylinConfig = CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
+
+ String createFlatTableMethod = kylinConfig.getCreateFlatHiveTableMethod();
+ if ("1".equals(createFlatTableMethod)) {
+ // create flat table first, then count and redistribute
+ jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, false, ""));
+ jobFlow.addTask(createRedistributeFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName));
+ } else if ("2".equals(createFlatTableMethod)) {
+ // count from source table first, and then redistribute, suitable for partitioned table
+ final String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()) + "/row_count";
+ jobFlow.addTask(createCountHiveTableStep(conf, flatDesc, jobFlow.getId(), rowCountOutputDir));
+ jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, true, rowCountOutputDir));
+ } else {
+ throw new IllegalArgumentException("Unknown value for kylin.hive.create.flat.table.method: " + createFlatTableMethod);
+ }
- jobFlow.addTask(createCountHiveTableStep(conf, flatDesc, jobFlow.getId(), rowCountOutputDir));
- jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName, rowCountOutputDir));
AbstractExecutable task = createLookupHiveViewMaterializationStep(jobFlow.getId());
if (task != null) {
jobFlow.addTask(task);
}
}
+ public static AbstractExecutable createRedistributeFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName) {
+ StringBuilder hiveInitBuf = new StringBuilder();
+ hiveInitBuf.append("USE ").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
+ hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
+
+ String rowCountOutputDir = JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
+
+ RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
+ step.setInitStatement(hiveInitBuf.toString());
+ step.setSelectRowCountStatement(JoinedFlatTable.generateSelectRowCountStatement(flatTableDesc, rowCountOutputDir));
+ step.setRowCountOutputDir(rowCountOutputDir);
+ step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatTableDesc));
+ CubingExecutableUtil.setCubeName(cubeName, step.getParams());
+ step.setName(ExecutableConstants.STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE);
+ return step;
+ }
+
+
public static AbstractExecutable createCountHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String rowCountOutputDir) {
final ShellExecutable step = new ShellExecutable();
@@ -174,17 +211,17 @@ public class HiveMRInput implements IMRInput {
return step;
}
- public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, String rowCountOutputDir) {
+ public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, boolean redistribute, String rowCountOutputDir) {
StringBuilder hiveInitBuf = new StringBuilder();
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
- String insertDataHqls;
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf);
+ String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf, redistribute);
CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
+ step.setUseRedistribute(redistribute);
step.setInitStatement(hiveInitBuf.toString());
step.setRowCountOutputDir(rowCountOutputDir);
step.setCreateTableStatement(useDatabaseHql + dropTableHql + createTableHql + insertDataHqls);
@@ -213,6 +250,126 @@ public class HiveMRInput implements IMRInput {
}
}
+ public static class RedistributeFlatHiveTableStep extends AbstractExecutable {
+ private final BufferedLogger stepLogger = new BufferedLogger(logger);
+
+ private void computeRowCount(CliCommandExecutor cmdExecutor) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(getInitStatement());
+ hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n");
+ hiveCmdBuilder.addStatement(getSelectRowCountStatement());
+ final String cmd = hiveCmdBuilder.build();
+
+ stepLogger.log("Compute row count of flat hive table, cmd: ");
+ stepLogger.log(cmd);
+
+ Pair<Integer, String> response = cmdExecutor.execute(cmd, stepLogger);
+ if (response.getFirst() != 0) {
+ throw new RuntimeException("Failed to compute row count of flat hive table");
+ }
+ }
+
+ private long readRowCountFromFile(Path file) throws IOException {
+ FileSystem fs = FileSystem.get(file.toUri(), HadoopUtil.getCurrentConfiguration());
+ InputStream in = fs.open(file);
+ try {
+ String content = IOUtils.toString(in);
+ return Long.valueOf(content.trim()); // strip the '\n' character
+
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
+ }
+
+ private int determineNumReducer(KylinConfig config) throws IOException {
+ computeRowCount(config.getCliCommandExecutor());
+
+ Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
+ long rowCount = readRowCountFromFile(rowCountFile);
+ int mapperInputRows = config.getHadoopJobMapperInputRows();
+
+ int numReducers = Math.round(rowCount / ((float) mapperInputRows));
+ numReducers = Math.max(1, numReducers);
+
+ stepLogger.log("total input rows = " + rowCount);
+ stepLogger.log("expected input rows per mapper = " + mapperInputRows);
+ stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
+
+ return numReducers;
+ }
+
+ private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(getInitStatement());
+ hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
+ hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n");
+ hiveCmdBuilder.addStatement(getRedistributeDataStatement());
+ final String cmd = hiveCmdBuilder.toString();
+
+ stepLogger.log("Redistribute table, cmd: ");
+ stepLogger.log(cmd);
+
+ Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger);
+ if (response.getFirst() != 0) {
+ throw new RuntimeException("Failed to redistribute flat hive table");
+ }
+ }
+
+ private KylinConfig getCubeSpecificConfig() {
+ String cubeName = CubingExecutableUtil.getCubeName(getParams());
+ CubeManager manager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube = manager.getCube(cubeName);
+ return cube.getConfig();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ KylinConfig config = getCubeSpecificConfig();
+
+ try {
+ int numReducers = determineNumReducer(config);
+ redistributeTable(config, numReducers);
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
+
+ } catch (Exception e) {
+ logger.error("job:" + getId() + " execute finished with exception", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+ }
+
+ public void setInitStatement(String sql) {
+ setParam("HiveInit", sql);
+ }
+
+ public String getInitStatement() {
+ return getParam("HiveInit");
+ }
+
+ public void setSelectRowCountStatement(String sql) {
+ setParam("HiveSelectRowCount", sql);
+ }
+
+ public String getSelectRowCountStatement() {
+ return getParam("HiveSelectRowCount");
+ }
+
+ public void setRedistributeDataStatement(String sql) {
+ setParam("HiveRedistributeData", sql);
+ }
+
+ public String getRedistributeDataStatement() {
+ return getParam("HiveRedistributeData");
+ }
+
+ public void setRowCountOutputDir(String rowCountOutputDir) {
+ setParam("rowCountOutputDir", rowCountOutputDir);
+ }
+
+ public String getRowCountOutputDir() {
+ return getParam("rowCountOutputDir");
+ }
+ }
+
public static class GarbageCollectionStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(GarbageCollectionStep.class);
[26/43] kylin git commit: KYLIN-1996 Keep original column order when
designing cube
Posted by sh...@apache.org.
KYLIN-1996 Keep original column order when designing cube
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e87c816d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e87c816d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e87c816d
Branch: refs/heads/v1.5.4-release2
Commit: e87c816dd7099290220ca09361f9ca04c36a317e
Parents: bf26114
Author: chenzhx <34...@qq.com>
Authored: Fri Sep 9 10:55:36 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Sep 9 15:52:48 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeDimensions.js | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e87c816d/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index 1663b5c..ab07451 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -67,20 +67,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var cols = $scope.getDimColumnsByTable(factTable);
// Initialize selected available.
- var factAvailable = {};
var factSelectAvailable = {};
for (var i = 0; i < cols.length; i++) {
cols[i].table = factTable;
cols[i].isLookup = false;
- factAvailable[cols[i].name] = cols[i];
-
// Default not selected and not disabled.
factSelectAvailable[cols[i].name] = {selected: false, disabled: false};
}
- $scope.availableColumns[factTable] = factAvailable;
+ $scope.availableColumns[factTable] = cols;
$scope.selectedColumns[factTable] = factSelectAvailable;
$scope.availableTables.push(factTable);
@@ -91,20 +88,17 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var cols2 = $scope.getDimColumnsByTable(lookups[j].table);
// Initialize selected available.
- var lookupAvailable = {};
var lookupSelectAvailable = {};
for (var k = 0; k < cols2.length; k++) {
cols2[k].table = lookups[j].table;
cols2[k].isLookup = true;
- lookupAvailable[cols2[k].name] = cols2[k];
-
// Default not selected and not disabled.
lookupSelectAvailable[cols2[k].name] = {selected: false, disabled: false};
}
- $scope.availableColumns[lookups[j].table] = lookupAvailable;
+ $scope.availableColumns[lookups[j].table] = cols2;
$scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
if($scope.availableTables.indexOf(lookups[j].table)==-1){
$scope.availableTables.push(lookups[j].table);
[22/43] kylin git commit: minor, better logging messages
Posted by sh...@apache.org.
minor, better logging messages
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6801695
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6801695
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6801695
Branch: refs/heads/v1.5.4-release2
Commit: d6801695240f5191716aec4cbd336470281fb20f
Parents: 01d033f
Author: Li Yang <li...@apache.org>
Authored: Thu Sep 8 17:02:17 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Sep 8 17:02:28 2016 +0800
----------------------------------------------------------------------
build/bin/setenv.sh | 2 +-
.../java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java | 2 +-
.../java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java | 2 +-
.../java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java | 4 ++--
.../org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/build/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
index d486672..317005b 100755
--- a/build/bin/setenv.sh
+++ b/build/bin/setenv.sh
@@ -20,7 +20,7 @@
# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
# uncomment following to for it to take effect
export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss256K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
# uncomment following to for it to take effect(the values need adjusting to fit your env)
# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
index 33b6f29..0b4ae40 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
@@ -56,7 +56,7 @@ public class BatchMergeJobBuilder extends JobBuilderSupport {
final String cuboidRootPath = getCuboidRootPath(jobId);
final List<CubeSegment> mergingSegments = cubeSegment.getCubeInstance().getMergingSegments(cubeSegment);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + cubeSegment);
final List<String> mergingSegmentIds = Lists.newArrayList();
final List<String> mergingCuboidPaths = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index 289cd48..129d525 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -48,7 +48,7 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
final String jobId = result.getId();
final List<CubeSegment> mergingSegments = cubeSegment.getCubeInstance().getMergingSegments(cubeSegment);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + cubeSegment);
final List<String> mergingSegmentIds = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index 1bd052d..7c2b3fd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -169,7 +169,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
public List<String> getMergingHTables() {
final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + seg);
final List<String> mergingHTables = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingHTables.add(merging.getStorageLocationIdentifier());
@@ -179,7 +179,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
public List<String> getMergingHDFSPaths() {
final List<CubeSegment> mergingSegments = ((CubeInstance) seg.getRealization()).getMergingSegments((CubeSegment) seg);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
+ Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge, target segment " + seg);
final List<String> mergingHDFSPaths = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingHDFSPaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6801695/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index 729635b..bdd3981 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -73,7 +73,7 @@ public class ZookeeperJobLock implements JobLock {
logger.warn("error acquire lock", e);
}
if (!hasLock) {
- logger.warn("fail to acquire lock, scheduler has not been started");
+ logger.warn("fail to acquire lock, scheduler has not been started; maybe another kylin process is still running?");
zkClient.close();
return false;
}
[03/43] kylin git commit: KYLIN-1767-WEB-UI-FOR-TOPN
Posted by sh...@apache.org.
KYLIN-1767-WEB-UI-FOR-TOPN
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e2970bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e2970bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e2970bc
Branch: refs/heads/v1.5.4-release2
Commit: 9e2970bc4a87d3848ba6677fd2d05c9ed86e15d9
Parents: 111e984
Author: zx chen <34...@qq.com>
Authored: Thu Sep 1 15:16:30 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Sep 2 19:42:22 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 25 ++++
webapp/app/js/controllers/cubeMeasures.js | 130 +++++++++++++++++++-
webapp/app/js/directives/directives.js | 29 ++++-
webapp/app/js/model/cubeDescModel.js | 6 +-
webapp/app/partials/cubeDesigner/measures.html | 77 ++++++++++--
5 files changed, 250 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index b620d7f..e2d0ab5 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -98,6 +98,31 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
return me_columns;
};
+ $scope.getGroupBYColumns = function () {
+ //metric from model
+ var me_columns = [];
+ var table_columns=[];
+ var groupby_columns=[];
+ var tableColumns = $scope.getColumnsByTable($scope.metaModel.model.fact_table);
+ if($scope.metaModel.model.metrics){
+ angular.forEach($scope.metaModel.model.metrics,function(metric,index){
+ me_columns.push(metric);
+ });
+ }
+ angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension){
+ if(dimension.table==$scope.metaModel.model.fact_table) {
+ table_columns.push(dimension.column);
+ }
+ });
+ angular.forEach(me_columns,function(column){
+ if(table_columns.indexOf(column)==-1) {
+ groupby_columns.push(column);
+ }
+ });
+
+ return groupby_columns;
+ };
+
$scope.getExtendedHostColumn = function(){
var me_columns = [];
//add cube dimension column for specific measure
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 9dec379..794d2b2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -19,7 +19,9 @@
'use strict';
KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
-
+ $scope.num=0;
+ $scope.convertedColumns=[];
+ $scope.groupby=[];
$scope.initUpdateMeasureStatus = function(){
$scope.updateMeasureStatus = {
isEdit:false,
@@ -38,6 +40,31 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(!!measure && measure.function.parameter.next_parameter){
$scope.nextPara.value = measure.function.parameter.next_parameter.value;
}
+ if($scope.newMeasure.function.expression=="TOP_N"){
+ $scope.convertedColumns=[];
+ for(var configuration in $scope.newMeasure.function.configuration) {
+ var _name=configuration.slice(14);
+ var item=$scope.newMeasure.function.configuration[configuration];
+ var _isFixedLength = item.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
+ var _isIntLength = item.substring(0,3) === "int"?"true":"false";//fixed_length:12
+ var _encoding = item;
+ var _valueLength = 0 ;
+ if(_isFixedLength !=="false"){
+ _valueLength = item.substring(13,item.length);
+ _encoding = "fixed_length";
+ }
+ if(_isIntLength!="false"){
+ _valueLength = item.substring(4,item.length);
+ _encoding = "int";
+ }
+ $scope.GroupBy = {
+ name:_name,
+ encoding:_encoding,
+ valueLength:_valueLength,
+ }
+ $scope.convertedColumns.push($scope.GroupBy);
+ };
+ }
};
@@ -103,9 +130,52 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.saveNewMeasure = function () {
- if ($scope.newMeasure.function.expression === 'TOP_N' && $scope.nextPara.value == "") {
- SweetAlert.swal('', '[TOP_N] Group by Column is required', 'warning');
- return false;
+ if ($scope.newMeasure.function.expression === 'TOP_N' ) {
+ if($scope.newMeasure.function.parameter.value == ""){
+ SweetAlert.swal('', '[TOP_N] ORDER|SUM by Column is required', 'warning');
+ return false;
+ }
+ if($scope.convertedColumns.length<1){
+ SweetAlert.swal('', '[TOP_N] Group by Column is required', 'warning');
+ return false;
+ }
+
+ var hasExisted = [];
+
+ for (var column in $scope.convertedColumns){
+ if(hasExisted.indexOf($scope.convertedColumns[column].name)==-1){
+ hasExisted.push($scope.convertedColumns[column].name);
+ }
+ else{
+ SweetAlert.swal('', 'The column named ['+$scope.convertedColumns[column].name+'] already exits.', 'warning');
+ return false;
+ }
+ if ($scope.convertedColumns[column].encoding == 'int' && ($scope.convertedColumns[column].valueLength < 1 || $scope.convertedColumns[column].valueLength > 8)) {
+ SweetAlert.swal('', 'int encoding column length should between 1 and 8.', 'warning');
+ return false;
+ }
+ }
+ $scope.nextPara.next_parameter={};
+ $scope.newMeasure.function.configuration={};
+ $scope.groupby($scope.nextPara);
+ angular.forEach($scope.convertedColumns,function(item){
+ var a='topn.encoding.'+item.name;
+ var encoding="";
+ if(item.encoding!=="dict" && item.encoding!=="date"&& item.encoding!=="time"){
+ if(item.encoding=="fixed_length" && item.valueLength){
+ encoding = "fixed_length:"+item.valueLength;
+ }
+ else if(item.encoding=="int" && item.valueLength){
+ encoding = "int:"+item.valueLength;
+ }else{
+ encoding = item.encoding;
+ }
+ }else{
+ encoding = item.encoding;
+ item.valueLength=0;
+ }
+ $scope.newMeasure.function.configuration[a]=encoding;
+ });
}
if ($scope.isNameDuplicated($scope.cubeMetaFrame.measures, $scope.newMeasure) == true) {
@@ -143,17 +213,67 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.nextPara = {
"type":"column",
"value":"",
- "next_parameter":null
+ "next_parameter":{}
}
if($scope.newMeasure){
$scope.newMeasure.function.parameter.next_parameter = null;
}
}
+ $scope.addNewGroupByColumn = function () {
+ $scope.nextGroupBy = {
+ name:null,
+ encoding:"dict",
+ valueLength:0,
+ }
+ $scope.convertedColumns.push($scope.nextGroupBy);
+
+ };
+
+ $scope.removeColumn = function(arr,index){
+ if (index > -1) {
+ arr.splice(index, 1);
+ }
+ };
+ $scope.refreshGroupBy=function (list,index,item) {
+ var encoding;
+ var name = item.name;
+ if(item.encoding=="dict" || item.encoding=="date"|| item.encoding=="time"){
+ item.valueLength=0;
+ }
+ };
+
+ $scope.groupby= function (next_parameter){
+ if($scope.num<$scope.convertedColumns.length-1){
+ next_parameter.value=$scope.convertedColumns[$scope.num].name;
+ next_parameter.type="column";
+ next_parameter.next_parameter={};
+ $scope.num++;
+ $scope.groupby(next_parameter.next_parameter);
+ }
+ else{
+ next_parameter.value=$scope.convertedColumns[$scope.num].name;
+ next_parameter.type="column";
+ next_parameter.next_parameter=null;
+ $scope.num=0;
+ return false;
+ }
+ }
+ $scope.converted = function (next_parameter) {
+ if (next_parameter != null) {
+ $scope.groupby.push(next_parameter.value);
+ converted(next_parameter.next_parameter)
+ }
+ else {
+ $scope.groupby.push(next_parameter.value);
+ return false;
+ }
+ }
//map right return type for param
$scope.measureReturnTypeUpdate = function(){
if($scope.newMeasure.function.expression == 'TOP_N'){
+ $scope.convertedColumns=[];
$scope.newMeasure.function.parameter.type= 'column';
$scope.newMeasure.function.returntype = "topn(100)";
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index 4b83865..f5051e8 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -295,7 +295,30 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
},
template:
'<li class="parent_li">Value:<b>{{nextpara.value}}</b>, Type:<b>{{ nextpara.type }}</b></li>' +
- '<parametertree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></parameterTree>',
+ '<parametertree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></parameterTree>',
+ compile: function(tElement, tAttr, transclude) {
+ var contents = tElement.contents().remove();
+ var compiledContents;
+ return function(scope, iElement, iAttr) {
+ if(!compiledContents) {
+ compiledContents = $compile(contents, transclude);
+ }
+ compiledContents(scope, function(clone, scope) {
+ iElement.append(clone);
+ });
+ };
+ }
+ };
+ }).directive("groupbytree", function($compile) {
+ return {
+ restrict: "E",
+ transclude: true,
+ scope: {
+ nextpara: '=',
+ },
+ template:
+ '<b>{{nextpara.value}}<b ng-if="nextpara.next_parameter!=null">,</b></b>'+
+ '<groupbytree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></groupbytree>',
compile: function(tElement, tAttr, transclude) {
var contents = tElement.contents().remove();
var compiledContents;
@@ -318,7 +341,9 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
},
template:
'<li class="parent_li">SUM|ORDER BY:<b>{{nextpara.value}}</b></b></li>' +
- '<li class="parent_li">GROUP BY:<b>{{nextpara.next_parameter.value}}</b></li>',
+ '<li class="parent_li">Group By:'+
+ '<groupbytree nextpara="nextpara.next_parameter"></groupbytree>'+
+ '</li>',
compile: function(tElement, tAttr, transclude) {
var contents = tElement.contents().remove();
var compiledContents;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index 57d7e6e..c981249 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -37,7 +37,8 @@ KylinApp.service('CubeDescModel', function () {
"type": "constant",
"value": "1",
"next_parameter":null
- }
+ },
+ "configuration":{}
}
}
],
@@ -74,7 +75,8 @@ KylinApp.service('CubeDescModel', function () {
"type": "",
"value": "",
"next_parameter":null
- }
+ },
+ "configuration":null
}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 53806b4..065b735 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -218,19 +218,80 @@
</div>
</div>
</div>
-
- <div class="form-group" ng-if="newMeasure.function.expression == 'TOP_N'">
+ <!--Group by Column-->
+ <div class="form-group" ng-if="newMeasure.function.expression == 'TOP_N'" >
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
<b>Group by Column</b>
</label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
- ng-model="nextPara.value"
- ng-options="column as column for column in getCommonMetricColumns()" >
- <option value=""></option>
- </select>
+ <div class="form-group large-popover" >
+ <div class="box-body">
+ <table style="margin-left:width:92%"
+ class="table table-hover table-bordered list">
+ <thead>
+ <tr>
+ <th>ID</th>
+ <th>Column</th>
+ <th>Encoding</th>
+ <th>Length</th>
+ <th ng-if="state.mode=='edit'"></th>
+ </tr>
+ </thead>
+
+ <tbody>
+
+ <tr ng-repeat="groupby_column in convertedColumns track by $index"
+ ng-class="state.mode=='edit'?'sort-item':''">
+
+ <td>
+ <!-- ID -->
+ <span class="ng-binding" ng-class="state.mode=='edit'?'badge':''">{{($index + 1)}}</span>
+ </td>
+ <!--Column Name -->
+ <td>
+ <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
+ ng-model="groupby_column.name"
+ ng-options="column as column for column in getGroupBYColumns()" style="width:323px;">
+ <option value="">--Select A Column--</option>
+ </select>
+ </td>
+ <!--Column Encoding -->
+ <td>
+ <select ng-if="state.mode=='edit'" style="width:180px;"
+ chosen ng-model="groupby_column.encoding"
+ ng-change="refreshGroupBy(convertedColumns,$index,groupby_column)"
+ ng-options="dt as dt for dt in store.supportedEncoding">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{groupby_column.encoding}}</span>
+
+ </td>
+ <td>
+ <!--Column Length -->
+ <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
+ tooltip="group by column length.." tooltip-trigger="focus"
+ ng-change="refreshGroupBy(convertedColumns,$index,groupby_column);"
+ ng-disabled="groupby_column.encoding=='dict'||groupby_column.encoding=='date'||groupby_column.encoding=='time'"
+ ng-model="groupby_column.valueLength" class="form-control">
+
+ <small class="help-block red" ng-show="state.mode=='edit' && groupby_column.encoding=='int' && (groupby_column.valueLength>8 || groupby_column.valueLength<1)">int encoding column length should between 1 and 8</small>
+ <span ng-if="state.mode=='view'">{{groupby_column.valueLength}}</span>
+ </td>
+ <td ng-if="state.mode=='edit'">
+ <button class="btn btn-xs btn-info"
+ ng-click="removeColumn(convertedColumns, $index)"><i
+ class="fa fa-minus"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <button class="btn btn-sm btn-info" style="margin-left:10px"
+ ng-click="addNewGroupByColumn()" ng-show="state.mode=='edit'">New Column<i class="fa fa-plus"></i>
+ </button>
</div>
+
</div>
</div>
[13/43] kylin git commit: KYLIN-1698 Also support int partition
column using format yyyyMMdd
Posted by sh...@apache.org.
KYLIN-1698 Also support int partition column using format yyyyMMdd
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb92f96f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb92f96f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb92f96f
Branch: refs/heads/v1.5.4-release2
Commit: eb92f96fc598a2add7a80a49768cd4f2d4184c76
Parents: ae72c25
Author: Li Yang <li...@apache.org>
Authored: Wed Sep 7 11:30:48 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Sep 7 11:30:48 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/PartitionDesc.java | 25 +++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb92f96f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 598e6d0..6487bfa 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -91,6 +91,14 @@ public class PartitionDesc {
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
+ public boolean partitionColumnIsYmdInt() {
+ if (partitionDateColumnRef == null)
+ return false;
+
+ DataType type = partitionDateColumnRef.getType();
+ return type.isInt();
+ }
+
public boolean partitionColumnIsTimeMillis() {
if (partitionDateColumnRef == null)
return false;
@@ -175,8 +183,10 @@ public class PartitionDesc {
String partitionDateColumnName = partDesc.getPartitionDateColumn();
String partitionTimeColumnName = partDesc.getPartitionTimeColumn();
- if (partDesc.partitionColumnIsTimeMillis()) {
- buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias);
+ if (partDesc.partitionColumnIsYmdInt()) {
+ buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias);
+ } else if (partDesc.partitionColumnIsTimeMillis()) {
+ buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias);
} else if (partitionDateColumnName != null && partitionTimeColumnName == null) {
buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), tableAlias);
} else if (partitionDateColumnName == null && partitionTimeColumnName != null) {
@@ -201,7 +211,7 @@ public class PartitionDesc {
return columnName;
}
- private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
if (startInclusive > 0) {
builder.append(partitionColumnName + " >= " + startInclusive);
@@ -210,6 +220,15 @@ public class PartitionDesc {
builder.append(partitionColumnName + " < " + endExclusive);
}
+ private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
+ if (startInclusive > 0) {
+ builder.append(partitionColumnName + " >= " + DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN));
+ builder.append(" AND ");
+ }
+ builder.append(partitionColumnName + " < " + DateFormat.formatToDateStr(endExclusive, DateFormat.COMPACT_DATE_PATTERN));
+ }
+
private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, Map<String, String> tableAlias) {
partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
if (startInclusive > 0) {
[05/43] kylin git commit: KYLIN-1834 dict offset can be upto 5 bytes,
trie dict is 2 GB at most
Posted by sh...@apache.org.
KYLIN-1834 dict offset can be upto 5 bytes, trie dict is 2 GB at most
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67bcec20
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67bcec20
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67bcec20
Branch: refs/heads/v1.5.4-release2
Commit: 67bcec20fb6fee502a5267dfa583bd07ca5edc8b
Parents: 7dae977
Author: Yang Li <li...@apache.org>
Authored: Sat Sep 3 13:13:00 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Sep 3 13:13:00 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BytesUtil.java | 2 +-
.../org/apache/kylin/dict/TrieDictionary.java | 30 +++---
.../kylin/dict/TrieDictionaryBuilder.java | 105 +++++++------------
.../apache/kylin/dict/TrieDictionaryTest.java | 74 ++++++++++---
4 files changed, 121 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index bda5c73..759ddbd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -98,7 +98,7 @@ public class BytesUtil {
/**
* No. bytes needed to store a value as big as the given
*/
- public static int sizeForValue(int maxValue) {
+ public static int sizeForValue(long maxValue) {
int size = 0;
while (maxValue > 0) {
size++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 03dc76a..aea9551 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -58,8 +58,8 @@ import com.google.common.base.Preconditions;
public class TrieDictionary<T> extends Dictionary<T> {
private static final long serialVersionUID = 1L;
- public static final byte[] HEAD_MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
- public static final int HEAD_SIZE_I = HEAD_MAGIC.length;
+ public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
+ public static final int MAGIC_SIZE_I = MAGIC.length;
public static final int BIT_IS_LAST_CHILD = 0x80;
public static final int BIT_IS_END_OF_VALUE = 0x40;
@@ -80,7 +80,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
transient private int nValues;
transient private int sizeOfId;
- transient private int childOffsetMask;
+ transient private long childOffsetMask;
transient private int firstByteOffset;
transient private boolean enableValueCache = true;
@@ -99,12 +99,12 @@ public class TrieDictionary<T> extends Dictionary<T> {
private void init(byte[] trieBytes) {
this.trieBytes = trieBytes;
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
+ if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
throw new IllegalArgumentException("Wrong file type (magic does not match)");
try {
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
this.headSize = headIn.readShort();
this.bodyLen = headIn.readInt();
this.sizeChildOffset = headIn.read();
@@ -118,7 +118,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
- this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
+ this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
} catch (Exception e) {
if (e instanceof RuntimeException)
@@ -229,7 +229,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
seq++;
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return roundSeqNo(roundingFlag, seq - 1, -1, seq); // input only partially matched
byte inpByte = inp[o];
@@ -253,6 +253,12 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
}
+ private int getChildOffset(int n) {
+ long offset = headSize + (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ assert offset < trieBytes.length;
+ return (int) offset;
+ }
+
private int roundSeqNo(int roundingFlag, int i, int j, int k) {
if (roundingFlag == 0)
return j;
@@ -338,7 +344,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return -1; // no child? corrupted dictionary!
int nValuesBeneath;
@@ -401,7 +407,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return;
@@ -446,14 +452,14 @@ public class TrieDictionary<T> extends Dictionary<T> {
@Override
public void readFields(DataInput in) throws IOException {
- byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE + Integer.SIZE];
+ byte[] headPartial = new byte[MAGIC.length + Short.SIZE + Integer.SIZE];
in.readFully(headPartial);
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
+ if (BytesUtil.compareBytes(MAGIC, 0, headPartial, 0, MAGIC.length) != 0)
throw new IllegalArgumentException("Wrong file type (magic does not match)");
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(headPartial, MAGIC_SIZE_I, headPartial.length - MAGIC_SIZE_I));
int headSize = headIn.readShort();
int bodyLen = headIn.readInt();
headIn.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 02da741..1271483 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -38,6 +38,8 @@ import org.apache.kylin.common.util.BytesUtil;
* @author yangli9
*/
public class TrieDictionaryBuilder<T> {
+
+ private static final int _2GB = 2000000000;
public static class Node {
public byte[] part;
@@ -112,8 +114,7 @@ public class TrieDictionaryBuilder<T> {
return;
}
- // if partially matched the current, split the current node, add the new
- // value, make a 3-way
+ // if partially matched the current, split the current node, add the new value, make a 3-way
if (i < n) {
Node c1 = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
@@ -128,8 +129,7 @@ public class TrieDictionaryBuilder<T> {
return;
}
- // out matched the current, binary search the next byte for a child node
- // to continue
+ // out matched the current, binary search the next byte for a child node to continue
byte lookfor = value[j];
int lo = 0;
int hi = node.children.size() - 1;
@@ -188,28 +188,21 @@ public class TrieDictionaryBuilder<T> {
public int mbpn_nNodes; // number of nodes in trie
public int mbpn_trieDepth; // depth of trie
public int mbpn_maxFanOut; // the maximum no. children
- public int mbpn_nChildLookups; // number of child lookups during lookup
- // every value once
- public int mbpn_nTotalFanOut; // the sum of fan outs during lookup every
- // value once
+ public long mbpn_nChildLookups; // number of child lookups during lookup every value once
+ public long mbpn_nTotalFanOut; // the sum of fan outs during lookup every value once
public int mbpn_sizeValueTotal; // the sum of value space in all nodes
public int mbpn_sizeNoValueBytes; // size of field noValueBytes
- public int mbpn_sizeNoValueBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int mbpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
- public int mbpn_footprint; // MBPN footprint in bytes
+ public int mbpn_sizeNoValueBeneath; // size of field noValuesBeneath, depends on cardinality
+ public int mbpn_sizeChildOffset; // size of field childOffset, points to first child in flattened array
+ public long mbpn_footprint; // MBPN footprint in bytes
// stats for one-byte-per-node as well, so there's comparison
public int obpn_sizeValue; // size of value per node, always 1
- public int obpn_sizeNoValuesBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int obpn_sizeChildCount; // size of field childCount, enables
- // binary search among children
- public int obpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
+ public int obpn_sizeNoValuesBeneath; // size of field noValuesBeneath, depends on cardinality
+ public int obpn_sizeChildCount; // size of field childCount, enables binary search among children
+ public int obpn_sizeChildOffset; // size of field childOffset, points to first child in flattened array
public int obpn_nNodes; // no. nodes in OBPN trie
- public int obpn_footprint; // OBPN footprint in bytes
+ public long obpn_footprint; // OBPN footprint in bytes
public void print() {
PrintStream out = System.out;
@@ -289,23 +282,12 @@ public class TrieDictionaryBuilder<T> {
s.obpn_sizeValue = 1;
s.obpn_sizeNoValuesBeneath = BytesUtil.sizeForValue(s.nValues);
s.obpn_sizeChildCount = 1;
- s.obpn_sizeChildOffset = 4; // MSB used as isEndOfValue flag
- s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total
- // number of compressed
- // bytes in OBPN
- s.obpn_footprint = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
+ s.obpn_sizeChildOffset = 5; // MSB used as isEndOfValue flag
+ s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total number of compressed bytes in OBPN
+ s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- int t = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2
- // because
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // flag
+ long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
+ if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2 because MSB of offset is used for isEndOfValue flag
s.obpn_sizeChildOffset--;
s.obpn_footprint = t;
} else
@@ -316,23 +298,11 @@ public class TrieDictionaryBuilder<T> {
s.mbpn_sizeValueTotal = s.nValueBytesCompressed;
s.mbpn_sizeNoValueBytes = 1;
s.mbpn_sizeNoValueBeneath = BytesUtil.sizeForValue(s.nValues);
- s.mbpn_sizeChildOffset = 4;
- s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
+ s.mbpn_sizeChildOffset = 5;
+ s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4
- // because
- // 2
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // &
- // isEndChild
- // flag
+ long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
+ if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
s.mbpn_sizeChildOffset--;
s.mbpn_footprint = t;
} else
@@ -415,8 +385,7 @@ public class TrieDictionaryBuilder<T> {
}
}
- completeParts.append(node.part);// by here the node.children may have
- // been changed
+ completeParts.append(node.part); // by here the node.children may have been changed
for (Node child : node.children) {
checkOverflowParts(child);
}
@@ -427,11 +396,13 @@ public class TrieDictionaryBuilder<T> {
* Flatten the trie into a byte array for a minimized memory footprint.
* Lookup remains fast. Cost is inflexibility to modify (becomes immutable).
*
- * Flattened node structure is HEAD + NODEs, for each node: - o byte, offset
- * to child node, o = stats.mbpn_sizeChildOffset - 1 bit, isLastChild flag,
- * the 1st MSB of o - 1 bit, isEndOfValue flag, the 2nd MSB of o - c byte,
- * number of values beneath, c = stats.mbpn_sizeNoValueBeneath - 1 byte,
- * number of value bytes - n byte, value bytes
+ * Flattened node structure is HEAD + NODEs, for each node:
+ * - o byte, offset to child node, o = stats.mbpn_sizeChildOffset
+ * - 1 bit, isLastChild flag, the 1st MSB of o
+ * - 1 bit, isEndOfValue flag, the 2nd MSB of o
+ * - c byte, number of values beneath, c = stats.mbpn_sizeNoValueBeneath
+ * - 1 byte, number of value bytes
+ * - n byte, value bytes
*/
public TrieDictionary<T> build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
@@ -445,15 +416,18 @@ public class TrieDictionaryBuilder<T> {
Stats stats = stats();
int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
int sizeChildOffset = stats.mbpn_sizeChildOffset;
+
+ if (stats.mbpn_footprint > _2GB)
+ throw new RuntimeException("Too big dictionary, dictionary cannot be bigger than 2GB");
// write head
byte[] head;
try {
ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
DataOutputStream headOut = new DataOutputStream(byteBuf);
- headOut.write(TrieDictionary.HEAD_MAGIC);
+ headOut.write(TrieDictionary.MAGIC);
headOut.writeShort(0); // head size, will back fill
- headOut.writeInt(stats.mbpn_footprint); // body size
+ headOut.writeInt((int) stats.mbpn_footprint); // body size
headOut.write(sizeChildOffset);
headOut.write(sizeNoValuesBeneath);
headOut.writeShort(baseId);
@@ -461,13 +435,12 @@ public class TrieDictionaryBuilder<T> {
headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
headOut.close();
head = byteBuf.toByteArray();
- BytesUtil.writeUnsigned(head.length, head, TrieDictionary.HEAD_SIZE_I, 2);
+ BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2);
} catch (IOException e) {
- throw new RuntimeException(e); // shall not happen, as we are
- // writing in memory
+ throw new RuntimeException(e); // shall not happen, as we are writing in memory
}
- byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
+ byte[] trieBytes = new byte[(int) stats.mbpn_footprint + head.length];
System.arraycopy(head, 0, trieBytes, 0, head.length);
LinkedList<Node> open = new LinkedList<Node>();
@@ -506,6 +479,8 @@ public class TrieDictionaryBuilder<T> {
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
int o = offset;
+ if (o > _2GB)
+ throw new IllegalStateException();
// childOffset
if (isLastChild)
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index 90283b8..a87d7cb 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -32,10 +32,10 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import java.util.Random;
import java.util.TreeSet;
@@ -44,12 +44,61 @@ import org.junit.Test;
public class TrieDictionaryTest {
public static void main(String[] args) throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- // InputStream is =
- // Util.getPackageResourceAsStream(TrieDictionaryTest.class,
- // "eng_com.dic");
- ArrayList<String> str = loadStrings(is);
- benchmarkStringDictionary(str);
+ int count = (int) (Integer.MAX_VALUE * 0.8 / 64);
+ benchmarkStringDictionary(new RandomStrings(count));
+ }
+
+ private static class RandomStrings implements Iterable<String> {
+ final private int size;
+
+ public RandomStrings(int size) {
+ this.size = size;
+ System.out.println("size = " + size);
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
+ Random rand = new Random(1000);
+ int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < size;
+ }
+
+ @Override
+ public String next() {
+ if (hasNext() == false)
+ throw new NoSuchElementException();
+
+ i++;
+ if (i % 1000000 == 0)
+ System.out.println(i);
+
+ return nextString();
+ }
+
+ private String nextString() {
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < 64; i++) {
+ int v = rand.nextInt(16);
+ char c;
+ if (v >= 0 && v <= 9)
+ c = (char) ('0' + v);
+ else
+ c = (char) ('a' + v - 10);
+ buf.append(c);
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
}
@Test
@@ -172,11 +221,11 @@ public class TrieDictionaryTest {
testStringDictionary(str, null);
}
- private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException {
+ private static void benchmarkStringDictionary(Iterable<String> str) throws IOException {
TrieDictionaryBuilder<String> b = newDictBuilder(str);
b.stats().print();
TrieDictionary<String> dict = b.build(0);
-
+
TreeSet<String> set = new TreeSet<String>();
for (String s : str) {
set.add(s);
@@ -205,13 +254,14 @@ public class TrieDictionaryTest {
// following jvm options may help
// -XX:CompileThreshold=1500
// -XX:+PrintCompilation
+ System.out.println("Benchmark awaitig...");
benchmark("Warm up", dict, set, map, strArray, array);
benchmark("Benchmark", dict, set, map, strArray, array);
}
private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
- int times = 10 * 1000 * 1000 / n; // run 10 million lookups
+ int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
byte[] valueBytes = new byte[dict.getSizeOfValue()];
long start;
@@ -259,7 +309,7 @@ public class TrieDictionaryTest {
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
System.out.println(timeIdToValueByDict);
-
+
return keep;
}
@@ -322,7 +372,7 @@ public class TrieDictionaryTest {
}
}
- private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) {
+ private static TrieDictionaryBuilder<String> newDictBuilder(Iterable<String> str) {
TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
for (String s : str)
b.addValue(s);
[11/43] kylin git commit: KYLIN-1962: minor,
fix Spring Security read Kylin properties
Posted by sh...@apache.org.
KYLIN-1962: minor, fix Spring Security read Kylin properties
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c35c859
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c35c859
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c35c859
Branch: refs/heads/v1.5.4-release2
Commit: 6c35c8594c34e7401a5adb43f649b1ed62e18841
Parents: 08f7dda
Author: Yiming Liu <li...@gmail.com>
Authored: Tue Sep 6 00:57:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Sep 6 12:00:47 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 38 ++++++++++----------
.../security/PasswordPlaceholderConfigurer.java | 20 +++++++++++
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 2 +-
3 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c35c859/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 241170a..f134ad4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -47,7 +47,7 @@ public class KylinConfig extends KylinConfigBase {
/** Kylin properties file name */
public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
- public static final String KYLIN_SECURITY_CONF_PROPERTIES_FILE = "kylin_account.properties";
+ public static final String KYLIN_ACCOUNT_CONF_PROPERTIES_FILE = "kylin_account.properties";
public static final String KYLIN_CONF = "KYLIN_CONF";
// static cached instances
@@ -205,11 +205,11 @@ public class KylinConfig extends KylinConfigBase {
return getKylinPropertiesFile(path);
}
- static File getKylinSecurityPropertiesFile() {
+ static File getKylinAccountPropertiesFile() {
String kylinConfHome = System.getProperty(KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
logger.info("Use KYLIN_CONF=" + kylinConfHome);
- return getKylinSecurityPropertiesFile(kylinConfHome);
+ return getKylinAccountPropertiesFile(kylinConfHome);
}
logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable");
@@ -219,10 +219,10 @@ public class KylinConfig extends KylinConfigBase {
throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
String path = kylinHome + File.separator + "conf";
- return getKylinSecurityPropertiesFile(path);
+ return getKylinAccountPropertiesFile(path);
}
- private static Properties getKylinProperties() {
+ public static Properties getKylinProperties() {
File propFile = getKylinPropertiesFile();
if (propFile == null || !propFile.exists()) {
logger.error("fail to locate " + KYLIN_CONF_PROPERTIES_FILE);
@@ -243,22 +243,22 @@ public class KylinConfig extends KylinConfigBase {
conf.putAll(propOverride);
}
- File securityPropFile = getKylinSecurityPropertiesFile();
- if (securityPropFile.exists()) {
- FileInputStream ois = new FileInputStream(securityPropFile);
- Properties propSecurity = new Properties();
- propSecurity.load(ois);
+ File accountPropFile = getKylinAccountPropertiesFile();
+ if (accountPropFile.exists()) {
+ FileInputStream ois = new FileInputStream(accountPropFile);
+ Properties propAccount = new Properties();
+ propAccount.load(ois);
IOUtils.closeQuietly(ois);
- conf.putAll(propSecurity);
+ conf.putAll(propAccount);
}
- File securityPropOverrideFile = new File(securityPropFile.getParentFile(), securityPropFile.getName() + ".override");
- if (securityPropOverrideFile.exists()) {
- FileInputStream ois = new FileInputStream(securityPropOverrideFile);
- Properties propSecurityOverride = new Properties();
- propSecurityOverride.load(ois);
+ File accountPropOverrideFile = new File(accountPropFile.getParentFile(), accountPropFile.getName() + ".override");
+ if (accountPropOverrideFile.exists()) {
+ FileInputStream ois = new FileInputStream(accountPropOverrideFile);
+ Properties propAccountOverride = new Properties();
+ propAccountOverride.load(ois);
IOUtils.closeQuietly(ois);
- conf.putAll(propSecurityOverride);
+ conf.putAll(propAccountOverride);
}
} catch (IOException e) {
@@ -282,12 +282,12 @@ public class KylinConfig extends KylinConfigBase {
return new File(path, KYLIN_CONF_PROPERTIES_FILE);
}
- private static File getKylinSecurityPropertiesFile(String path) {
+ private static File getKylinAccountPropertiesFile(String path) {
if (path == null) {
return null;
}
- return new File(path, KYLIN_SECURITY_CONF_PROPERTIES_FILE);
+ return new File(path, KYLIN_ACCOUNT_CONF_PROPERTIES_FILE);
}
public static void setSandboxEnvIfPossible() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c35c859/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 5381b14..092d73a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -18,13 +18,21 @@
package org.apache.kylin.rest.security;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.charset.Charset;
import java.util.Properties;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
/**
@@ -38,7 +46,19 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
*/
private static byte[] key = { 0x74, 0x68, 0x69, 0x73, 0x49, 0x73, 0x41, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79 };
+ /**
+ * The PasswordPlaceholderConfigurer will read Kylin properties as the Spring resource
+ */
public PasswordPlaceholderConfigurer() {
+ Resource[] resources = new Resource[1];
+ Properties prop = KylinConfig.getKylinProperties();
+ StringWriter writer = new StringWriter();
+ prop.list(new PrintWriter(writer));
+ String propString = writer.getBuffer().toString();
+ IOUtils.closeQuietly(writer);
+ InputStream is = IOUtils.toInputStream(propString, Charset.defaultCharset());
+ resources[0] = new InputStreamResource(is);
+ this.setLocations(resources);
}
public static String encrypt(String strToEncrypt) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c35c859/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index e77ac3b..f93aaf2 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -184,7 +184,7 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
File[] confFiles = srcConfDir.listFiles();
if (confFiles != null) {
for (File confFile : confFiles) {
- if (!KylinConfig.KYLIN_SECURITY_CONF_PROPERTIES_FILE.equals(confFile.getName())) {
+ if (!KylinConfig.KYLIN_ACCOUNT_CONF_PROPERTIES_FILE.equals(confFile.getName())) {
FileUtils.copyFileToDirectory(confFile, destConfDir);
}
}
[07/43] kylin git commit: KYLIN-1984: Disable compress in default
configuration
Posted by sh...@apache.org.
KYLIN-1984: Disable compress in default configuration
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99d0d753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99d0d753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99d0d753
Branch: refs/heads/v1.5.4-release2
Commit: 99d0d7535ae307a3b34a8ffd23b5d2d50559eea5
Parents: 663820f
Author: Yiming Liu <li...@gmail.com>
Authored: Sun Sep 4 17:22:20 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Sep 4 21:15:48 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 4 ++--
build/conf/kylin_hive_conf.xml | 13 ++++++++++++-
build/conf/kylin_job_conf.xml | 15 ++++++++++++---
build/conf/kylin_job_conf_inmem.xml | 15 ++++++++++++---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 5 +++--
6 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5607336..c20488a 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -54,8 +54,8 @@ kylin.storage.cleanup.time.threshold=172800000
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin
-# Compression codec for htable, valid value [snappy, lzo, gzip, lz4]
-kylin.hbase.default.compression.codec=snappy
+# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
+kylin.hbase.default.compression.codec=none
# HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020
# Leave empty if hbase running on same cluster with hive and mapreduce
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index 3d6109b..30c4feb 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -39,17 +39,28 @@
<description>enable map-side join</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
+ -->
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 877e82f..96b806c 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/build/conf/kylin_job_conf_inmem.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf_inmem.xml b/build/conf/kylin_job_conf_inmem.xml
index 1cd809d..fea2f68 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -31,26 +31,35 @@
<description>Compress map outputs</description>
</property>
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
-
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-
+ -->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1390e24..f0c91da 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -615,7 +615,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getHbaseDefaultCompressionCodec() {
- return getOptional("kylin.hbase.default.compression.codec", "");
+ return getOptional("kylin.hbase.default.compression.codec", "none");
}
public String getHbaseDefaultEncoding() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99d0d753/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index fe65598..9b487a7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -183,8 +183,9 @@ public class CubeHTableUtil {
cf.setCompressionType(Algorithm.LZ4);
break;
}
+ case "none":
default: {
- logger.info("hbase will not user any compression algorithm to compress data");
+ logger.info("hbase will not use any compression algorithm to compress data");
cf.setCompressionType(Algorithm.NONE);
}
}
@@ -194,7 +195,7 @@ public class CubeHTableUtil {
DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
cf.setDataBlockEncoding(encoding);
} catch (Exception e) {
- logger.info("hbase will not use any encoding");
+ logger.info("hbase will not use any encoding", e);
cf.setDataBlockEncoding(DataBlockEncoding.NONE);
}
[27/43] kylin git commit: KYLIN-1922 optimize needStorageAggregation
check logic and make sure self-termination in coprocessor works
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
new file mode 100644
index 0000000..0cdfa7e
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.gridtable.CubeCodeSystem;
+import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.GTBuilder;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.GridTable;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
+import org.apache.kylin.gridtable.GTInfo.Builder;
+import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class DictGridTableTest extends LocalFileMetadataTestCase {
+
+ private GridTable table;
+ private GTInfo info;
+ private CompareTupleFilter timeComp0;
+ private CompareTupleFilter timeComp1;
+ private CompareTupleFilter timeComp2;
+ private CompareTupleFilter timeComp3;
+ private CompareTupleFilter timeComp4;
+ private CompareTupleFilter timeComp5;
+ private CompareTupleFilter timeComp6;
+ private CompareTupleFilter ageComp1;
+ private CompareTupleFilter ageComp2;
+ private CompareTupleFilter ageComp3;
+ private CompareTupleFilter ageComp4;
+
+ @After
+ public void after() throws Exception {
+
+ this.cleanupTestMetadata();
+ }
+
+ @Before
+ public void setup() throws IOException {
+
+ this.createTestMetadata();
+
+ table = newTestTable();
+ info = table.getInfo();
+
+ timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14"));
+ timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
+ timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
+ timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
+ timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15"));
+ timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14"));
+ ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
+ ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
+ ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
+ ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
+
+ }
+
+ @Test
+ public void verifySegmentSkipping() {
+
+ ByteArray segmentStart = enc(info, 0, "2015-01-14");
+ ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+ assertEquals(segmentStart, segmentStartX);
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp2, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp4, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
+ assertEquals("[[null, 10, null, null, null], [1421193600000, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
+ assertEquals(0, r.get(0).fuzzyKeys.size());
+ }
+ {
+ //skip FALSE filter
+ LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+ {
+ //TRUE or FALSE filter
+ LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[null, null]-[null, null]", r.get(0).toString());
+ }
+ {
+ //TRUE or other filter
+ LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[null, null]-[null, null]", r.get(0).toString());
+ }
+ }
+
+ @Test
+ public void verifySegmentSkipping2() {
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());//scan range are [close,close]
+ }
+ }
+
+ @Test
+ public void verifyScanRangePlanner() {
+
+ // flatten or-and & hbase fuzzy value
+ {
+ LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
+ assertEquals("[[null, 10, null, null, null], [null, 20, null, null, null]]", r.get(0).fuzzyKeys.toString());
+ }
+
+ // pre-evaluate ever false
+ {
+ LogicalTupleFilter filter = and(timeComp1, timeComp2);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(0, r.size());
+ }
+
+ // pre-evaluate ever true
+ {
+ LogicalTupleFilter filter = or(timeComp1, ageComp4);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals("[[null, null]-[null, null]]", r.toString());
+ }
+
+ // merge overlap range
+ {
+ LogicalTupleFilter filter = or(timeComp1, timeComp3);
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals("[[null, null]-[null, null]]", r.toString());
+ }
+
+ // merge too many ranges
+ {
+ LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
+ CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
+ List<GTScanRange> r = planner.planScanRanges();
+ assertEquals(3, r.size());
+ assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
+ assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString());
+ assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString());
+ planner.setMaxScanRanges(2);
+ List<GTScanRange> r2 = planner.planScanRanges();
+ assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString());
+ }
+ }
+
+ @Test
+ public void verifyFirstRow() throws IOException {
+ doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", //
+ "[1421193600000, 30, Luke, 10, 10.5]", //
+ "[1421280000000, 20, Dong, 10, 10.5]", //
+ "[1421280000000, 20, Jason, 10, 10.5]", //
+ "[1421280000000, 30, Xu, 10, 10.5]", //
+ "[1421366400000, 20, Mahone, 10, 10.5]", //
+ "[1421366400000, 20, Qianhao, 10, 10.5]", //
+ "[1421366400000, 30, George, 10, 10.5]", //
+ "[1421366400000, 30, Shaofeng, 10, 10.5]", //
+ "[1421452800000, 10, Kejia, 10, 10.5]");
+ }
+
+ //for testing GTScanRequest serialization and deserialization
+ public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
+ ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
+ GTScanRequest.serializer.serialize(origin, buffer);
+ buffer.flip();
+ GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer);
+
+ Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs());
+ Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01);
+ return sGTScanRequest;
+ }
+
+ @Test
+ public void verifyScanWithUnevaluatableFilter() throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
+ LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
+ LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);
+
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }).setFilterPushDown(filter).createGTScanRequest();
+
+ // note the unEvaluatable column 1 in filter is added to group by
+ assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
+ }
+
+ @Test
+ public void verifyScanWithEvaluatableFilter() throws IOException {
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }).setFilterPushDown(filter).createGTScanRequest();
+ // note the evaluatable column 1 in filter is added to returned columns but not in group by
+ assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+
+ doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]");
+ }
+
+ @Test
+ public void testFilterScannerPerf() throws IOException {
+ GridTable table = newTestPerfTable();
+ GTInfo info = table.getInfo();
+
+ CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ FilterResultCache.ENABLED = false;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = true;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = false;
+ testFilterScannerPerfInner(table, info, filter);
+ FilterResultCache.ENABLED = true;
+ testFilterScannerPerfInner(table, info, filter);
+ }
+
+ @SuppressWarnings("unused")
+ private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException {
+ long start = System.currentTimeMillis();
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(filter).createGTScanRequest();
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ i++;
+ }
+ scanner.close();
+ long end = System.currentTimeMillis();
+ System.out.println((end - start) + "ms with filter cache enabled=" + FilterResultCache.ENABLED + ", " + i + " rows");
+ }
+
+ @Test
+ public void verifyConvertFilterConstants1() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants2() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1<"9" round up to $1<"10"
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants3() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1<="9" round down to FALSE
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString());
+ }
+
+ @Test
+ public void verifyConvertFilterConstants4() {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
+ TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
+ LogicalTupleFilter filter = and(fComp1, fComp2);
+
+ List<TblColRef> colMapping = Lists.newArrayList();
+ colMapping.add(extColA);
+ colMapping.add(extColB);
+
+ // $1 in ("9", "10", "15") has only "10" left
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString());
+ }
+
+ private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
+ System.out.println(req);
+ IGTScanner scanner = table.scan(req);
+ int i = 0;
+ for (GTRecord r : scanner) {
+ System.out.println(r);
+ if (verifyRows == null || i >= verifyRows.length) {
+ Assert.fail();
+ }
+ assertEquals(verifyRows[i], r.toString());
+ i++;
+ }
+ scanner.close();
+ }
+
+ public static ByteArray enc(GTInfo info, int col, String value) {
+ ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
+ info.getCodeSystem().encodeColumnValue(col, value, buf);
+ return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
+ }
+
+ public static ExtractTupleFilter unevaluatable(TblColRef col) {
+ ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
+ r.addChild(new ColumnTupleFilter(col));
+ return r;
+ }
+
+ public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
+ CompareTupleFilter result = new CompareTupleFilter(op);
+ result.addChild(new ColumnTupleFilter(col));
+ result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
+ return result;
+ }
+
+ public static LogicalTupleFilter and(TupleFilter... children) {
+ return logic(FilterOperatorEnum.AND, children);
+ }
+
+ public static LogicalTupleFilter or(TupleFilter... children) {
+ return logic(FilterOperatorEnum.OR, children);
+ }
+
+ public static LogicalTupleFilter not(TupleFilter child) {
+ return logic(FilterOperatorEnum.NOT, child);
+ }
+
+ public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+ LogicalTupleFilter result = new LogicalTupleFilter(op);
+ for (TupleFilter c : children) {
+ result.addChild(c);
+ }
+ return result;
+ }
+
+ public static GridTable newTestTable() throws IOException {
+ GTInfo info = newInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.close();
+
+ return table;
+ }
+
+ static GridTable newTestPerfTable() throws IOException {
+ GTInfo info = newInfo();
+ GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GridTable table = new GridTable(info, store);
+
+ GTRecord r = new GTRecord(table.getInfo());
+ GTBuilder builder = table.rebuild();
+
+ for (int i = 0; i < 100000; i++) {
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+
+ for (int j = 0; j < 10; j++)
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ }
+ builder.close();
+
+ return table;
+ }
+
+ static GTInfo newInfo() {
+ Builder builder = GTInfo.builder();
+ builder.setCodeSystem(newDictCodeSystem());
+ builder.setColumns( //
+ DataType.getType("timestamp"), //
+ DataType.getType("integer"), //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("bigint"), //
+ DataType.getType("decimal") //
+ );
+ builder.setPrimaryKey(setOf(0, 1));
+ builder.setColumnPreferIndex(setOf(0));
+ builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
+ builder.enableRowBlock(4);
+ GTInfo info = builder.build();
+ return info;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static CubeCodeSystem newDictCodeSystem() {
+ DimensionEncoding[] dimEncs = new DimensionEncoding[3];
+ dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
+ dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString());
+ return new CubeCodeSystem(dimEncs);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfString() {
+ TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("Dong");
+ builder.addValue("George");
+ builder.addValue("Jason");
+ builder.addValue("Kejia");
+ builder.addValue("Luke");
+ builder.addValue("Mahone");
+ builder.addValue("Qianhao");
+ builder.addValue("Shaofeng");
+ builder.addValue("Xu");
+ builder.addValue("Yang");
+ return builder.build(0);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static Dictionary newDictionaryOfInteger() {
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ builder.addValue("10");
+ builder.addValue("20");
+ builder.addValue("30");
+ builder.addValue("40");
+ builder.addValue("50");
+ builder.addValue("60");
+ builder.addValue("70");
+ builder.addValue("80");
+ builder.addValue("90");
+ builder.addValue("100");
+ return builder.build(0);
+ }
+
+ public static ImmutableBitSet setOf(int... values) {
+ BitSet set = new BitSet();
+ for (int i : values)
+ set.set(i);
+ return new ImmutableBitSet(set);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 375b198..fc2fd52 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -22,13 +22,16 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
@@ -37,18 +40,26 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
public class ITKylinQueryTest extends KylinTestBase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@BeforeClass
public static void setUp() throws Exception {
printInfo("setUp in ITKylinQueryTest");
@@ -108,10 +119,52 @@ public class ITKylinQueryTest extends KylinTestBase {
return "";
}
- @Ignore("this is only for debug")
+ @Test
+ public void testTimeoutQuery() throws Exception {
+
+ thrown.expect(SQLException.class);
+
+ //should not break at table duplicate check, should fail at model duplicate check
+ thrown.expectCause(new BaseMatcher<Throwable>() {
+ @Override
+ public boolean matches(Object item) {
+ if (item instanceof GTScanSelfTerminatedException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ }
+ });
+
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ BackdoorToggles.setToggles(toggles);
+
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
+
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
+ BackdoorToggles.cleanToggles();
+ }
+
+ //don't try to ignore this test, try to clean your "temp" folder
@Test
public void testTempQuery() throws Exception {
+ PRINT_RESULT = true;
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/temp", null, true);
+ PRINT_RESULT = false;
}
@Ignore
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/kylin-it/src/test/resources/query/sql_timeout/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
new file mode 100644
index 0000000..3b9a837
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 1200
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
index 75533cd..5f21351 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
@@ -26,4 +26,5 @@ public enum CoprocessorBehavior {
SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
SCAN_FILTER_AGGR, //aggregate the result. Will return results
SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 07a3cc3..5b48351 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -43,8 +43,8 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTScanRange;
import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -106,7 +106,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
- logger.debug("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
+ logger.info("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard();
short shardNum = shardNumAndBaseShard.getFirst();
@@ -146,7 +146,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
rawScanBufferSize *= 4;
}
}
- scanRequest.setGTScanRanges(Lists.<GTScanRange> newArrayList());//since raw scans are sent to coprocessor, we don't need to duplicate sending it
+ scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
@@ -248,7 +248,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
if (abnormalFinish[0]) {
- Throwable ex = new RuntimeException(logHeader + "The coprocessor thread stopped itself due to scan timeout or scan threshold(check region server log), failing current query...");
+ Throwable ex = new GTScanSelfTerminatedException(logHeader + "The coprocessor thread stopped itself due to scan timeout or scan threshold(check region server log), failing current query...");
logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout
epResultItr.notifyCoprocException(ex);
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index a359d19..f1e5dab 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -213,7 +213,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize());
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT, cubeSeg.getRowKeyPreambleSize(), false);
IGTScanner rawScanner = store.scan(scanRequest);
final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 7d48c1a..442963f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -18,20 +18,22 @@
package org.apache.kylin.storage.hbase.cube.v2;
+import java.util.Iterator;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
class ExpectedSizeIterator implements Iterator<byte[]> {
private static final Logger logger = LoggerFactory.getLogger(ExpectedSizeIterator.class);
@@ -48,22 +50,24 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
this.expectedSize = expectedSize;
this.queue = new ArrayBlockingQueue<byte[]>(expectedSize);
+ StringBuilder sb = new StringBuilder();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+
this.rpcTimeout = hconf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
this.timeout = this.rpcTimeout * hconf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
- logger.info("rpc timeout is {} and after multiply retry times become {}", this.rpcTimeout, this.timeout);
- this.timeout = Math.max(this.timeout, 5 * 60000);
+ sb.append("rpc timeout is " + this.rpcTimeout + " and after multiply retry times becomes " + this.timeout);
+
this.timeout *= KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
+ sb.append(" after multiply kylin.query.cube.visit.timeout.times becomes " + this.timeout);
+
+ logger.info(sb.toString());
if (BackdoorToggles.getQueryTimeout() != -1) {
this.timeout = BackdoorToggles.getQueryTimeout();
+ logger.info("rpc timeout is overwritten to " + this.timeout);
}
- this.timeout *= 1.1; // allow for some delay
-
- logger.info("Final Timeout for ExpectedSizeIterator is: " + this.timeout);
-
- this.timeoutTS = System.currentTimeMillis() + this.timeout;
+ this.timeoutTS = System.currentTimeMillis() + 2 * this.timeout;//longer timeout than coprocessor so that query thread will not timeout faster than coprocessor
}
@Override
@@ -85,9 +89,14 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
}
if (coprocException != null) {
- throw new RuntimeException("Error in coprocessor", coprocException);
+ if (coprocException instanceof GTScanSelfTerminatedException)
+ throw (GTScanSelfTerminatedException) coprocException;
+ else
+ throw new RuntimeException("Error in coprocessor",coprocException);
+
} else if (ret == null) {
- throw new RuntimeException("Timeout visiting cube!");
+ throw new RuntimeException("Timeout visiting cube! Check why coprocessor exception is not sent back? In coprocessor Self-termination is checked every " + //
+ GTScanRequest.terminateCheckInterval + " scanned rows, the configured timeout(" + timeout + ") cannot support this many scans?");
} else {
return ret;
}
@@ -110,7 +119,7 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
}
public long getRpcTimeout() {
- return this.rpcTimeout;
+ return this.timeout;
}
public void notifyCoprocException(Throwable ex) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 4b9b4fa..1d8ad79 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -43,13 +43,15 @@ public class HBaseReadonlyStore implements IGTStore {
private List<Pair<byte[], byte[]>> hbaseColumns;
private List<List<Integer>> hbaseColumnsToGT;
private int rowkeyPreambleSize;
+ private boolean withDelay = false;
- public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize) {
+ public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize, boolean withDelay) {
this.cellListIterator = cellListIterator;
this.info = gtScanRequest.getInfo();
this.hbaseColumns = hbaseColumns;
this.hbaseColumnsToGT = hbaseColumnsToGT;
this.rowkeyPreambleSize = rowkeyPreambleSize;
+ this.withDelay = withDelay;
}
@Override
@@ -95,6 +97,13 @@ public class HBaseReadonlyStore implements IGTStore {
@Override
public boolean hasNext() {
+ if (withDelay) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
return cellListIterator.hasNext();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index b29d0d1..064d100 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -170,7 +170,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
@SuppressWarnings("checkstyle:methodlength")
@Override
- public void visitCube(final RpcController controller, CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
+ public void visitCube(final RpcController controller, final CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
List<RegionScanner> regionScanners = Lists.newArrayList();
HRegion region = null;
@@ -241,7 +241,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
if (behavior.ordinal() < CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
- scanReq.setAggCacheMemThreshold(0); // disable mem check if so told
+ scanReq.disableAggCacheMemCheck(); // disable mem check if so told
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
@@ -266,7 +266,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter);
}
- if (counter % 100000 == 1) {
+ if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
logger.info("Scanned " + counter + " rows from HBase.");
}
counter++;
@@ -284,7 +284,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize());
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, //
+ request.getRowkeyPreambleSize(), CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(request.getBehavior()));
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, //
@@ -299,14 +300,9 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
for (GTRecord oneRecord : finalScanner) {
- if (finalRowCount > storagePushDownLimit) {
- logger.info("The finalScanner aborted because storagePushDownLimit is satisfied");
- break;
- }
-
- if (finalRowCount % 100000 == 1) {
+ if (finalRowCount % GTScanRequest.terminateCheckInterval == 1) {
if (System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("finalScanner timeouts after scanned " + finalRowCount);
+ throw new GTScanTimeoutException("finalScanner timeouts after contributed " + finalRowCount);
}
}
@@ -319,7 +315,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
outputStream.write(buffer.array(), 0, buffer.position());
+
finalRowCount++;
+
+ //if it's doing storage aggr, then should rely on GTAggregateScanner's limit check
+ if (!scanReq.isDoingStorageAggregation() && finalRowCount >= storagePushDownLimit) {
+ //read one more record than limit
+ logger.info("The finalScanner aborted because storagePushDownLimit is satisfied");
+ break;
+ }
}
} catch (GTScanTimeoutException e) {
scanNormalComplete.setValue(false);
[35/43] kylin git commit: Revert "remove unnecessary raw measure"
Posted by sh...@apache.org.
Revert "remove unnecessary raw measure"
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7a3fdf5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7a3fdf5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7a3fdf5
Branch: refs/heads/v1.5.4-release2
Commit: d7a3fdf57acb6fbabf94669aaca869e47d89aa13
Parents: a05f111
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Sep 10 14:40:25 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Sep 10 14:42:46 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/topn/TopNMeasureType.java | 2 +
.../test_case_data/localmeta/cube_desc/ssb.json | 409 ++++++-------
.../test_kylin_cube_with_slr_desc.json | 389 +++++-------
...st_kylin_cube_with_view_inner_join_desc.json | 388 +++++-------
...est_kylin_cube_with_view_left_join_desc.json | 388 +++++-------
.../test_kylin_cube_without_slr_desc.json | 61 +-
...t_kylin_cube_without_slr_left_join_desc.json | 584 +++++++++----------
.../test_streaming_table_cube_desc.json | 245 ++++----
8 files changed, 1048 insertions(+), 1418 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 0756056..01eb90c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -274,9 +274,11 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (sum.isSum() == false)
return false;
+
if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0)
return false;
+
TblColRef sumCol = sum.getParameter().getColRefs().get(0);
return sumCol.equals(topnNumCol);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index 4903979..d3ea10b 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -1,256 +1,179 @@
{
- "uuid": "5c44df30-daec-486e-af90-927bf7851057",
- "name": "ssb",
- "description": "",
- "dimensions": [
- {
- "name": "SSB.PART_DERIVED",
- "table": "SSB.PART",
- "column": null,
- "derived": [
- "P_MFGR",
- "P_CATEGORY",
- "P_BRAND"
- ]
- },
- {
- "name": "C_CITY",
- "table": "SSB.CUSTOMER",
- "column": "C_CITY",
- "derived": null
- },
- {
- "name": "C_REGION",
- "table": "SSB.CUSTOMER",
- "column": "C_REGION",
- "derived": null
- },
- {
- "name": "C_NATION",
- "table": "SSB.CUSTOMER",
- "column": "C_NATION",
- "derived": null
- },
- {
- "name": "S_CITY",
- "table": "SSB.SUPPLIER",
- "column": "S_CITY",
- "derived": null
- },
- {
- "name": "S_REGION",
- "table": "SSB.SUPPLIER",
- "column": "S_REGION",
- "derived": null
- },
- {
- "name": "S_NATION",
- "table": "SSB.SUPPLIER",
- "column": "S_NATION",
- "derived": null
- },
- {
- "name": "D_YEAR",
- "table": "SSB.DATES",
- "column": "D_YEAR",
- "derived": null
- },
- {
- "name": "D_YEARMONTH",
- "table": "SSB.DATES",
- "column": "D_YEARMONTH",
- "derived": null
- },
- {
- "name": "D_YEARMONTHNUM",
- "table": "SSB.DATES",
- "column": "D_YEARMONTHNUM",
- "derived": null
- },
- {
- "name": "D_WEEKNUMINYEAR",
- "table": "SSB.DATES",
- "column": "D_WEEKNUMINYEAR",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "_COUNT_",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
+ "name" : "ssb",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "C_CITY",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_CITY",
+ "derived" : null
+ }, {
+ "name" : "C_REGION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_REGION",
+ "derived" : null
+ }, {
+ "name" : "C_NATION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_NATION",
+ "derived" : null
+ }, {
+ "name" : "S_CITY",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_CITY",
+ "derived" : null
+ }, {
+ "name" : "S_REGION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_REGION",
+ "derived" : null
+ }, {
+ "name" : "S_NATION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_REVENUE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "LO_REVENUE",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_SUPPLYCOST",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "LO_SUPPLYCOST",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "TOTAL_V_REVENUE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "V_REVENUE",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_V_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "V_REVENUE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "LO_PARTKEY",
- "encoding": "dict"
- },
- {
- "column": "C_CITY",
- "encoding": "dict"
- },
- {
- "column": "C_REGION",
- "encoding": "dict"
- },
- {
- "column": "C_NATION",
- "encoding": "dict"
- },
- {
- "column": "S_CITY",
- "encoding": "dict"
- },
- {
- "column": "S_REGION",
- "encoding": "dict"
- },
- {
- "column": "S_NATION",
- "encoding": "dict"
- },
- {
- "column": "D_YEAR",
- "encoding": "dict"
- },
- {
- "column": "D_YEARMONTH",
- "encoding": "dict"
- },
- {
- "column": "D_YEARMONTHNUM",
- "encoding": "dict"
- },
- {
- "column": "D_WEEKNUMINYEAR",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
},
- "signature": "5iV8LVYs+PmVUju8QNQ5TQ==",
- "last_modified": 1457503036686,
- "model_name": "ssb",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "_COUNT_",
- "TOTAL_REVENUE",
- "TOTAL_SUPPLYCOST",
- "TOTAL_V_REVENUE"
- ]
- }
- ]
- }
- ]
+ "signature" : "5iV8LVYs+PmVUju8QNQ5TQ==",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "LO_PARTKEY",
- "C_CITY",
- "C_REGION",
- "C_NATION",
- "S_CITY",
- "S_REGION",
- "S_NATION",
- "D_YEAR",
- "D_YEARMONTH",
- "D_YEARMONTHNUM",
- "D_WEEKNUMINYEAR"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "C_REGION",
- "C_NATION",
- "C_CITY"
- ],
- [
- "S_REGION",
- "S_NATION",
- "S_CITY"
- ],
- [
- "D_YEARMONTH",
- "D_YEARMONTHNUM",
- "D_WEEKNUMINYEAR"
- ]
- ],
- "mandatory_dims": [
- "D_YEAR"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ "D_YEAR" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": [],
- "status_need_notify": [],
- "partition_date_start": 694224000000,
- "partition_date_end": 3153600000000,
- "auto_merge_time_ranges": [
- 604800000,
- 2419200000
- ],
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
- "override_kylin_properties": {
- "kylin.hbase.default.compression.codec": "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid": "true"
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 694224000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.hbase.default.compression.codec" : "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index f62d196..4064fcb 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -1,245 +1,172 @@
{
- "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_with_slr_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- },
- {
- "name": "SELLER_ID",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "SELLER_ID",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+
+ "name" : "test_kylin_cube_with_slr_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "SELLER_ID",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "seller_id",
- "encoding": "int:4",
- "isShardBy": true
- },
- {
- "column": "cal_dt",
- "encoding": "dict"
- },
- {
- "column": "leaf_categ_id",
- "encoding": "fixed_length:18"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "fixed_length:12"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "seller_id",
+ "encoding" : "int:4",
+ "isShardBy" : true
+ }, {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "fixed_length:18"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801271,
- "model_name": "test_kylin_inner_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801271,
+ "model_name" : "test_kylin_inner_join_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name",
- "seller_id",
- "slr_segment_cd"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "seller_id"
- ],
- "joint_dims": [
- [
- "lstg_format_name",
- "lstg_site_id",
- "slr_segment_cd"
- ]
- ]
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : ["seller_id"],
+ "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
- "partition_date_start": 0
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "partition_date_start" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
index e3a3e70..d4c64b5 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
@@ -1,249 +1,169 @@
{
- "uuid": "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_with_view_inner_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.V_TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_with_view_inner_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.V_TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_inner_join_view_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_inner_join_view_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ }]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
+ }
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
index b17fbff..0388c0e 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
@@ -1,249 +1,169 @@
{
- "uuid": "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_with_view_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.V_TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_with_view_left_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.V_TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_left_join_view_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_left_join_view_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
+ } ]
+ }]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
+ }
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index d185175..28328e4 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -1,4 +1,5 @@
{
+
"uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
"name": "test_kylin_cube_without_slr_desc",
"description": null,
@@ -159,19 +160,54 @@
"returntype": "extendedcolumn(100)"
},
"dependent_measure_ref": null
- },
- {
- "name": "PRICE_RAW",
- "function": {
- "expression": "RAW",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT",
+ "next_parameter" : null
},
- "returntype": "raw"
+ "returntype" : "raw"
},
- "dependent_measure_ref": null
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LEAF_CATEG_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LEAF_CATEG_ID",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
}
],
"rowkey": {
@@ -234,6 +270,9 @@
"item_count_sum",
"SITE_EXTENDED_1",
"SITE_EXTENDED_2",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
"PRICE_RAW"
]
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 2aea1a8..ca1b35c 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -1,357 +1,293 @@
{
- "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_without_slr_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": "{FK}",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "{FK}",
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ]
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
+ "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "test_kylin_cube_without_slr_left_join_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "EDW.TEST_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SITE_ID",
+ "table" : "EDW.TEST_SITES",
+ "column" : "{FK}",
+ "derived" : [ "SITE_NAME", "CRE_USER" ]
+ }, {
+ "name" : "SELLER_TYPE_CD",
+ "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "column" : "{FK}",
+ "derived" : [ "SELLER_TYPE_DESC" ]
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": "{FK}",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
},
- {
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": "{FK}",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_CNT_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bitmap"
},
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SITE_NAME_BITMAP",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SITE_NAME",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bitmap"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_FORMAT_CNT",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
},
- "dependent_measure_ref": null
+ "returntype" : "hllc(10)"
},
- {
- "name": "SELLER_CNT_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- },
- "returntype": "bitmap"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
},
- "dependent_measure_ref": null
+ "returntype" : "topn(100)",
+ "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
},
- {
- "name": "SITE_NAME_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SITE_NAME",
- "next_parameter": null
- },
- "returntype": "bitmap"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "CAL_DT_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "CAL_DT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "hllc(10)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "topn(100)",
- "configuration": {
- "topn.encoding.SELLER_ID": "int:4"
- }
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "LEAF_CATEG_ID_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LEAF_CATEG_ID",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "raw"
},
- {
- "name": "PRICE_RAW",
- "function": {
- "expression": "RAW",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": null
- },
- "returntype": "raw"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "PRICE_RAW",
+ "function" : {
+ "expression" : "RAW",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "dictionaries": [
+ "returntype" : "raw"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "dictionaries" : [
{
- "column": "SITE_NAME",
+ "column" : "SITE_NAME",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
}
],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "encoding": "dict"
- },
- {
- "column": "leaf_categ_id",
- "encoding": "dict"
- },
- {
- "column": "meta_categ_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl2_name",
- "encoding": "dict"
- },
- {
- "column": "categ_lvl3_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_format_name",
- "encoding": "dict"
- },
- {
- "column": "lstg_site_id",
- "encoding": "dict"
- },
- {
- "column": "slr_segment_cd",
- "encoding": "dict"
- }
- ]
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "cal_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "slr_segment_cd",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801311,
- "model_name": "test_kylin_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum",
- "PRICE_RAW"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "seller_cnt_bitmap",
- "site_name_bitmap",
- "seller_format_cnt"
- ]
- }
- ]
- },
- {
- "name": "f3",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801311,
+ "model_name" : "test_kylin_left_join_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "seller_cnt_bitmap", "site_name_bitmap", "seller_format_cnt"]
+ } ]
+ }, {
+ "name" : "f3",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "top_seller" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "lstg_format_name",
- "lstg_site_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": [
- [
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ]
- ]
- }
- },
- {
- "includes": [
- "cal_dt",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "leaf_categ_id",
- "meta_categ_name"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ]
- ],
- "mandatory_dims": [
- "cal_dt"
- ],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
}
- ],
- "notify_list": null,
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ }, {
+ "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ "cal_dt" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"override_kylin_properties": {
"kylin.job.cubing.inmem.sampling.hll.precision": "16"
},
http://git-wip-us.apache.org/repos/asf/kylin/blob/d7a3fdf5/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index f2c4e72..ef10c1e 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -1,155 +1,118 @@
{
- "uuid": "901ed15e-7769-4c66-b7ae-fbdc971cd192",
- "name": "test_streaming_table_cube_desc",
- "description": "",
- "dimensions": [
- {
- "name": "DEFAULT.STREAMING_TABLE.SITE",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "SITE",
- "derived": null
- },
- {
- "name": "DEFAULT.STREAMING_TABLE.ITM",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "ITM",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "DAY_START",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "HOUR_START",
- "derived": null
- },
- {
- "name": "TIME",
- "table": "DEFAULT.STREAMING_TABLE",
- "column": "MINUTE_START",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "_COUNT_",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd192",
+
+ "name" : "test_streaming_table_cube_desc",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "DEFAULT.STREAMING_TABLE.SITE",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "SITE",
+ "derived" : null
+ }, {
+ "name" : "DEFAULT.STREAMING_TABLE.ITM",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "ITM",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "DAY_START",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "HOUR_START",
+ "derived" : null
+ }, {
+ "name" : "TIME",
+ "table" : "DEFAULT.STREAMING_TABLE",
+ "column" : "MINUTE_START",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "GMV",
- "next_parameter": null
- },
- "returntype": "decimal(19,6)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "GMV",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,6)"
},
- {
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT",
- "next_parameter": null
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "ITEM_COUNT_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "ITEM_COUNT",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "DAY_START",
- "encoding": "dict"
- },
- {
- "column": "HOUR_START",
- "encoding": "dict"
- },
- {
- "column": "MINUTE_START",
- "encoding": "dict"
- },
- {
- "column": "SITE",
- "encoding": "dict"
- },
- {
- "column": "ITM",
- "encoding": "dict"
- }
- ]
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "DAY_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "HOUR_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "MINUTE_START",
+ "encoding" : "dict"
+ }, {
+ "column" : "SITE",
+ "encoding" : "dict"
+ }, {
+ "column" : "ITM",
+ "encoding" : "dict"
+ } ]
},
- "signature": null,
- "last_modified": 1448959801314,
- "model_name": "test_streaming_table_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "_COUNT_",
- "GMV_SUM",
- "ITEM_COUNT_SUM"
- ]
- }
- ]
- }
- ]
+ "signature" : null,
+ "last_modified" : 1448959801314,
+ "model_name" : "test_streaming_table_model_desc",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "GMV_SUM", "ITEM_COUNT_SUM" ]
+ } ]
+ } ]
},
- "aggregation_groups": [
- {
- "includes": [
- "DAY_START",
- "HOUR_START",
- "ITM",
- "MINUTE_START",
- "SITE"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "DAY_START",
- "HOUR_START",
- "MINUTE_START"
- ]
- ],
- "mandatory_dims": [],
- "joint_dims": []
- }
+ "aggregation_groups" : [ {
+ "includes" : [ "DAY_START", "HOUR_START", "ITM", "MINUTE_START", "SITE" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "DAY_START", "HOUR_START", "MINUTE_START" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
}
- ],
+ } ],
"override_kylin_properties": {
"kylin.cube.algorithm": "inmem"
},
- "notify_list": [],
- "status_need_notify": [],
- "auto_merge_time_ranges": null,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
"partition_date_start": 0
}
\ No newline at end of file
[15/43] kylin git commit: KYLIN-1926-FK-PK-data-type-matching
Posted by sh...@apache.org.
KYLIN-1926-FK-PK-data-type-matching
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ede67e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ede67e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ede67e3
Branch: refs/heads/v1.5.4-release2
Commit: 4ede67e3f5a3bb1439d34263edb09c7ebf334675
Parents: eb92f96
Author: chenzhx <34...@qq.com>
Authored: Wed Sep 7 10:48:46 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 11:39:28 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeModel.js | 45 ++++++++++++--------
.../app/partials/modelDesigner/data_model.html | 5 ++-
2 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ede67e3/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
index be931f3..b20a829 100644
--- a/webapp/app/js/controllers/cubeModel.js
+++ b/webapp/app/js/controllers/cubeModel.js
@@ -59,7 +59,10 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
join: {
type: '',
primary_key: [],
- foreign_key: []
+ foreign_key: [],
+ isCompatible:[],
+ pk_type:[],
+ fk_type:[]
}
};
};
@@ -82,7 +85,6 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
backdrop: 'static',
scope: $scope
});
-
modalInstance.result.then(function () {
if (!$scope.lookupState.editing) {
$scope.doneAddLookup();
@@ -163,14 +165,37 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
};
+ $scope.changeKey = function(index){
+ var fact_table = modelsManager.selectedModel.fact_table;
+ var lookup_table = $scope.newLookup.table;
+ var pk_column = $scope.newLookup.join.primary_key[index];
+ var fk_column = $scope.newLookup.join.foreign_key[index];
+ if(pk_column!=='null'&&fk_column!=='null'){
+ $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(pk_column,lookup_table);
+ $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(fk_column,fact_table);
+ if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
+ $scope.newLookup.join.isCompatible[index]=false;
+ }else{
+ $scope.newLookup.join.isCompatible[index]=true;
+ }
+
+ }
+ }
+
$scope.addNewJoin = function(){
$scope.newLookup.join.primary_key.push("null");
$scope.newLookup.join.foreign_key.push("null");
+ $scope.newLookup.join.fk_type.push("null");
+ $scope.newLookup.join.pk_type.push("null");
+ $scope.newLookup.join.isCompatible.push(true);
};
$scope.removeJoin = function($index){
$scope.newLookup.join.primary_key.splice($index,1);
$scope.newLookup.join.foreign_key.splice($index,1);
+ $scope.newLookup.join.fk_type.splice($index,1);
+ $scope.newLookup.join.pk_type.splice($index,1);
+ $scope.newLookup.join.isCompatible.splice($index,1);
};
$scope.resetParams = function () {
@@ -195,22 +220,6 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
}
}
- //column type validate
- var fact_table = modelsManager.selectedModel.fact_table;
- var lookup_table = $scope.newLookup.table;
-
- for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
- var pk_column = $scope.newLookup.join.primary_key[i];
- var fk_column = $scope.newLookup.join.foreign_key[i];
- if(pk_column!=='null'&&fk_column!=='null'){
- var pk_type = TableModel.getColumnType(pk_column,lookup_table);
- var fk_type = TableModel.getColumnType(fk_column,fact_table);
- if(pk_type!==fk_type){
- errors.push(" Column Type incompatible "+pk_column+"["+pk_type+"]"+","+fk_column+"["+fk_type+"].");
- }
- }
- }
-
var errorInfo = "";
angular.forEach(errors,function(item){
errorInfo+="\n"+item;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ede67e3/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index d1a9cdd..662d0d9 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -157,13 +157,13 @@
<div ng-repeat="joinIndex in [] | range: newLookup.join.primary_key.length">
<div>
<select style="width: 45%" chosen data-placeholder="Fact Table Column"
- ng-model="newLookup.join.foreign_key[$index]"
+ ng-model="newLookup.join.foreign_key[$index]" ng-change="changeKey($index)"
ng-options="columns.name as columns.name for columns in getColumnsByTable(modelsManager.selectedModel.fact_table)" >
<option value=""></option>
</select>
<b>=</b>
<select style="width: 45%" chosen data-placeholder="Lookup Table Column"
- ng-model="newLookup.join.primary_key[$index]"
+ ng-model="newLookup.join.primary_key[$index]" ng-change="changeKey($index)"
ng-options="columns.name as columns.name for columns in getColumnsByTable(newLookup.table)" >
<option value=""></option>
</select>
@@ -173,6 +173,7 @@
</button>
</div>
<div class="space-4"></div>
+ <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}],{{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}].</b></small>
</div>
</div>
</div>
[28/43] kylin git commit: KYLIN-1922 optimize needStorageAggregation
check logic and make sure self-termination in coprocessor works
Posted by sh...@apache.org.
KYLIN-1922 optimize needStorageAggregation check logic and make sure self-termination in coprocessor works
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e38557b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e38557b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e38557b4
Branch: refs/heads/v1.5.4-release2
Commit: e38557b4d1cd1d42fe042e5500020cbfaba2d80b
Parents: e87c816
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 15:57:25 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 16:42:33 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 15 +-
.../apache/kylin/cube/RawQueryLastHacker.java | 7 +-
.../cube/gridtable/CubeScanRangePlanner.java | 340 ----------
.../kylin/gridtable/GTAggregateScanner.java | 10 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 6 +-
.../GTScanExceedThresholdException.java | 2 +-
.../apache/kylin/gridtable/GTScanRequest.java | 34 +-
.../GTScanSelfTerminatedException.java | 26 +
.../kylin/gridtable/GTScanTimeoutException.java | 2 +-
.../gridtable/AggregationCacheSpillTest.java | 6 +-
.../kylin/gridtable/DictGridTableTest.java | 617 ------------------
.../storage/gtrecord/CubeScanRangePlanner.java | 357 +++++++++++
.../storage/gtrecord/CubeSegmentScanner.java | 14 +-
.../gtrecord/GTCubeStorageQueryBase.java | 36 +-
.../storage/gtrecord/DictGridTableTest.java | 626 +++++++++++++++++++
.../apache/kylin/query/ITKylinQueryTest.java | 55 +-
.../resources/query/sql_timeout/query01.sql | 19 +
.../common/coprocessor/CoprocessorBehavior.java | 1 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 8 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/ExpectedSizeIterator.java | 39 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 11 +-
.../coprocessor/endpoint/CubeVisitService.java | 26 +-
23 files changed, 1197 insertions(+), 1062 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f0c91da..2ac9d48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,8 +481,8 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
- public int getCubeVisitTimeoutTimes() {
- return Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
+ public float getCubeVisitTimeoutTimes() {
+ return Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
}
public int getBadQueryStackTraceDepth() {
@@ -545,15 +545,6 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false"));
}
- public String getQueryStorageVisitPlanner() {
- return this.getOptional("kylin.query.storage.visit.planner", "org.apache.kylin.cube.gridtable.CubeScanRangePlanner");
- }
-
- // for test only
- public void setQueryStorageVisitPlanner(String v) {
- setProperty("kylin.query.storage.visit.planner", v);
- }
-
public int getQueryScanFuzzyKeyMax() {
return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
}
@@ -573,7 +564,7 @@ abstract public class KylinConfigBase implements Serializable {
public boolean getQueryMetricsEnabled() {
return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false"));
}
-
+
public int[] getQueryMetricsPercentilesIntervals() {
String[] dft = { "60", "300", "3600" };
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 63ddac5..50c644e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -44,13 +44,14 @@ public class RawQueryLastHacker {
// We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
for (TblColRef col : cubeDesc.listAllColumns()) {
- if (col.getTable().equals(sqlDigest.factTable) && (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll)) {
- sqlDigest.allColumns.add(col);
+ if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll) {
+ if (col.getTable().equals(sqlDigest.factTable))
+ sqlDigest.allColumns.add(col);
}
}
for (TblColRef col : sqlDigest.allColumns) {
- if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
+ if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) {
// For dimension columns, take them as group by columns.
sqlDigest.groupbyColumns.add(col);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
deleted file mode 100644
index a937045..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.common.FuzzyValueCombination;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.gridtable.GTScanRange;
-import org.apache.kylin.gridtable.GTScanRequest;
-import org.apache.kylin.gridtable.GTScanRequestBuilder;
-import org.apache.kylin.gridtable.GTUtil;
-import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-public class CubeScanRangePlanner extends ScanRangePlannerBase {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
-
- protected int maxScanRanges;
- protected int maxFuzzyKeys;
-
- //non-GT
- protected CubeSegment cubeSegment;
- protected CubeDesc cubeDesc;
- protected Cuboid cuboid;
-
- public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
- Collection<FunctionDesc> metrics) {
-
- this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
- this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
-
- this.cubeSegment = cubeSegment;
- this.cubeDesc = cubeSegment.getCubeDesc();
- this.cuboid = cuboid;
-
- Set<TblColRef> filterDims = Sets.newHashSet();
- TupleFilter.collectColumns(filter, filterDims);
-
- this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
- CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
-
- IGTComparator comp = gtInfo.getCodeSystem().getComparator();
- //start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
- //stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
- //start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
-
- //replace the constant values in filter to dictionary codes
- this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
-
- this.gtDimensions = mapping.makeGridTableColumns(dimensions);
- this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
- this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
- this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
-
- if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
- int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
- if (index >= 0) {
- SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
- this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
- this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
- this.gtPartitionCol = gtInfo.colRef(index);
- }
- }
-
- }
-
- /**
- * constrcut GTScanRangePlanner with incomplete information. only be used for UT
- * @param info
- * @param gtStartAndEnd
- * @param gtPartitionCol
- * @param gtFilter
- */
- public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
-
- this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
- this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
-
- this.gtInfo = info;
-
- IGTComparator comp = gtInfo.getCodeSystem().getComparator();
- //start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
- //stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
- //start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
-
- this.gtFilter = gtFilter;
- this.gtStartAndEnd = gtStartAndEnd;
- this.gtPartitionCol = gtPartitionCol;
- }
-
- public GTScanRequest planScanRequest() {
- GTScanRequest scanRequest;
- List<GTScanRange> scanRanges = this.planScanRanges();
- if (scanRanges != null && scanRanges.size() != 0) {
- scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).createGTScanRequest();
- } else {
- scanRequest = null;
- }
- return scanRequest;
- }
-
- /**
- * Overwrite this method to provide smarter storage visit plans
- * @return
- */
- public List<GTScanRange> planScanRanges() {
- TupleFilter flatFilter = flattenToOrAndFilter(gtFilter);
-
- List<Collection<ColumnRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter);
-
- List<GTScanRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
- for (Collection<ColumnRange> andDimRanges : orAndDimRanges) {
- GTScanRange scanRange = newScanRange(andDimRanges);
- if (scanRange != null)
- scanRanges.add(scanRange);
- }
-
- List<GTScanRange> mergedRanges = mergeOverlapRanges(scanRanges);
- mergedRanges = mergeTooManyRanges(mergedRanges, maxScanRanges);
-
- return mergedRanges;
- }
-
- private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
- Set<TblColRef> ret = Sets.newHashSet();
- for (TblColRef col : input) {
- if (cubeDesc.hasHostColumn(col)) {
- for (TblColRef host : cubeDesc.getHostInfo(col).columns) {
- ret.add(host);
- }
- } else {
- ret.add(col);
- }
- }
- return ret;
- }
-
- protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
- GTRecord pkStart = new GTRecord(gtInfo);
- GTRecord pkEnd = new GTRecord(gtInfo);
- Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
-
- List<GTRecord> fuzzyKeys;
-
- for (ColumnRange range : andDimRanges) {
- if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
- int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
- int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
-
- if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
- } else {
- logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
- gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
- return null;
- }
- }
-
- int col = range.column.getColumnDesc().getZeroBasedIndex();
- if (!gtInfo.getPrimaryKey().get(col))
- continue;
-
- pkStart.set(col, range.begin);
- pkEnd.set(col, range.end);
-
- if (range.valueSet != null && !range.valueSet.isEmpty()) {
- fuzzyValues.put(col, range.valueSet);
- }
- }
-
- fuzzyKeys =
-
- buildFuzzyKeys(fuzzyValues);
- return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
- }
-
- private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
- ArrayList<GTRecord> result = Lists.newArrayList();
-
- if (fuzzyValueSet.isEmpty())
- return result;
-
- // debug/profiling purpose
- if (BackdoorToggles.getDisableFuzzyKey()) {
- logger.info("The execution of this query will not use fuzzy key");
- return result;
- }
-
- List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKeys);
-
- for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
-
- // BitSet bitSet = new BitSet(gtInfo.getColumnCount());
- // for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
- // bitSet.set(entry.getKey());
- // }
- GTRecord fuzzy = new GTRecord(gtInfo);
- for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
- fuzzy.set(entry.getKey(), entry.getValue());
- }
-
- result.add(fuzzy);
- }
- return result;
- }
-
- protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
- if (ranges.size() <= 1) {
- return ranges;
- }
-
- // sort ranges by start key
- Collections.sort(ranges, new Comparator<GTScanRange>() {
- @Override
- public int compare(GTScanRange a, GTScanRange b) {
- return rangeStartComparator.compare(a.pkStart, b.pkStart);
- }
- });
-
- // merge the overlap range
- List<GTScanRange> mergedRanges = new ArrayList<GTScanRange>();
- int mergeBeginIndex = 0;
- GTRecord mergeEnd = ranges.get(0).pkEnd;
- for (int index = 1; index < ranges.size(); index++) {
- GTScanRange range = ranges.get(index);
-
- // if overlap, swallow it
- if (rangeStartEndComparator.compare(range.pkStart, mergeEnd) <= 0) {
- mergeEnd = rangeEndComparator.max(mergeEnd, range.pkEnd);
- continue;
- }
-
- // not overlap, split here
- GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, index));
- mergedRanges.add(mergedRange);
-
- // start new split
- mergeBeginIndex = index;
- mergeEnd = range.pkEnd;
- }
-
- // don't miss the last range
- GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, ranges.size()));
- mergedRanges.add(mergedRange);
-
- return mergedRanges;
- }
-
- private GTScanRange mergeKeyRange(List<GTScanRange> ranges) {
- GTScanRange first = ranges.get(0);
- if (ranges.size() == 1)
- return first;
-
- GTRecord start = first.pkStart;
- GTRecord end = first.pkEnd;
- List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
-
- boolean hasNonFuzzyRange = false;
- for (GTScanRange range : ranges) {
- hasNonFuzzyRange = hasNonFuzzyRange || range.fuzzyKeys.isEmpty();
- newFuzzyKeys.addAll(range.fuzzyKeys);
- end = rangeEndComparator.max(end, range.pkEnd);
- }
-
- // if any range is non-fuzzy, then all fuzzy keys must be cleared
- // also too many fuzzy keys will slow down HBase scan
- if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKeys) {
- newFuzzyKeys.clear();
- }
-
- return new GTScanRange(start, end, newFuzzyKeys);
- }
-
- protected List<GTScanRange> mergeTooManyRanges(List<GTScanRange> ranges, int maxRanges) {
- if (ranges.size() <= maxRanges) {
- return ranges;
- }
-
- // TODO: check the distance between range and merge the large distance range
- List<GTScanRange> result = new ArrayList<GTScanRange>(1);
- GTScanRange mergedRange = mergeKeyRange(ranges);
- result.add(mergedRange);
- return result;
- }
-
- public int getMaxScanRanges() {
- return maxScanRanges;
- }
-
- public void setMaxScanRanges(int maxScanRanges) {
- this.maxScanRanges = maxScanRanges;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index ccf4895..db38484 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -138,7 +138,10 @@ public class GTAggregateScanner implements IGTScanner {
long count = 0;
for (GTRecord r : inputScanner) {
- count++;
+ //check deadline
+ if (count % GTScanRequest.terminateCheckInterval == 1 && System.currentTimeMillis() > deadline) {
+ throw new GTScanTimeoutException("Timeout in GTAggregateScanner with scanned count " + count);
+ }
if (getNumOfSpills() == 0) {
//check limit
@@ -152,10 +155,7 @@ public class GTAggregateScanner implements IGTScanner {
aggrCache.aggregate(r, Integer.MAX_VALUE);
}
- //check deadline
- if (count % 10000 == 1 && System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("Timeout in GTAggregateScanner with scanned count " + count);
- }
+ count++;
}
logger.info("GTAggregateScanner input rows: " + count);
return aggrCache.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index 31a9599..f1f84af 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -132,12 +132,12 @@ public class GTFilterScanner implements IGTScanner {
}
// cache the last one input and result, can reuse because rowkey are ordered, and same input could come in small group
- static class FilterResultCache {
+ public static class FilterResultCache {
static final int CHECKPOINT = 10000;
static final double HIT_RATE_THRESHOLD = 0.5;
- static boolean ENABLED = true; // enable cache by default
+ public static boolean ENABLED = true; // enable cache by default
- boolean enabled = ENABLED;
+ public boolean enabled = ENABLED;
ImmutableBitSet colsInFilter;
int count;
int hit;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
index dd57e90..ba75962 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanExceedThresholdException.java
@@ -18,7 +18,7 @@
package org.apache.kylin.gridtable;
-public class GTScanExceedThresholdException extends RuntimeException {
+public class GTScanExceedThresholdException extends GTScanSelfTerminatedException {
public GTScanExceedThresholdException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 4cfba1b..5d27028 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -42,6 +42,8 @@ import com.google.common.collect.Sets;
public class GTScanRequest {
private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
+ //it's not necessary to increase the checkInterval to very large because the check cost is not high
+ public static final int terminateCheckInterval = 1000;
private GTInfo info;
private List<GTScanRange> ranges;
@@ -55,13 +57,16 @@ public class GTScanRequest {
private ImmutableBitSet aggrGroupBy;
private ImmutableBitSet aggrMetrics;
private String[] aggrMetricsFuncs;//
-
+
// hint to storage behavior
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
private int storageScanRowNumThreshold;
private int storagePushDownLimit;
+ // runtime computed fields
+ private transient boolean doingStorageAggregation = false;
+
GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, boolean allowStorageAggregation, //
double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit) {
@@ -169,6 +174,7 @@ public class GTScanRequest {
logger.info("pre aggregation is not beneficial, skip it");
} else if (this.hasAggregation()) {
logger.info("pre aggregating results before returning");
+ this.doingStorageAggregation = true;
result = new GTAggregateScanner(result, this, deadline);
} else {
logger.info("has no aggregation, skip it");
@@ -178,6 +184,10 @@ public class GTScanRequest {
}
+ public boolean isDoingStorageAggregation() {
+ return doingStorageAggregation;
+ }
+
//touch every byte of the cell so that the cost of scanning will be truly reflected
private int lookAndForget(IGTScanner scanner) {
byte meaninglessByte = 0;
@@ -215,8 +225,8 @@ public class GTScanRequest {
return ranges;
}
- public void setGTScanRanges(List<GTScanRange> ranges) {
- this.ranges = ranges;
+ public void clearScanRanges() {
+ this.ranges = Lists.newArrayList();
}
public ImmutableBitSet getSelectedColBlocks() {
@@ -251,10 +261,6 @@ public class GTScanRequest {
return allowStorageAggregation;
}
- public void setAllowStorageAggregation(boolean allowStorageAggregation) {
- this.allowStorageAggregation = allowStorageAggregation;
- }
-
public double getAggCacheMemThreshold() {
if (aggCacheMemThreshold < 0)
return 0;
@@ -262,28 +268,18 @@ public class GTScanRequest {
return aggCacheMemThreshold;
}
- public void setAggCacheMemThreshold(double gb) {
- this.aggCacheMemThreshold = gb;
+ public void disableAggCacheMemCheck() {
+ this.aggCacheMemThreshold = 0;
}
public int getStorageScanRowNumThreshold() {
return storageScanRowNumThreshold;
}
- public void setStorageScanRowNumThreshold(int storageScanRowNumThreshold) {
- logger.info("storageScanRowNumThreshold is set to " + storageScanRowNumThreshold);
- this.storageScanRowNumThreshold = storageScanRowNumThreshold;
- }
-
public int getStoragePushDownLimit() {
return this.storagePushDownLimit;
}
- public void setStoragePushDownLimit(int limit) {
- logger.info("storagePushDownLimit is set to " + storagePushDownLimit);
- this.storagePushDownLimit = limit;
- }
-
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
new file mode 100644
index 0000000..4775ac6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+public class GTScanSelfTerminatedException extends RuntimeException {
+
+ public GTScanSelfTerminatedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
index e92dae3..17a8d02 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanTimeoutException.java
@@ -18,7 +18,7 @@
package org.apache.kylin.gridtable;
-public class GTScanTimeoutException extends RuntimeException {
+public class GTScanTimeoutException extends GTScanSelfTerminatedException {
public GTScanTimeoutException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index b5f6de7..38b8c90 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -84,8 +84,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
};
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).createGTScanRequest();
- scanRequest.setAggCacheMemThreshold(0.5);
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
@@ -127,8 +126,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
};
// all-in-mem testcase
- GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).createGTScanRequest();
- scanRequest.setAggCacheMemThreshold(0.5);
+ GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
deleted file mode 100644
index 7b6d3fa..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.gridtable.CubeCodeSystem;
-import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
-import org.apache.kylin.dict.NumberDictionaryBuilder;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.apache.kylin.dict.TrieDictionaryBuilder;
-import org.apache.kylin.dimension.DictionaryDimEnc;
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.gridtable.GTFilterScanner.FilterResultCache;
-import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ExtractTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class DictGridTableTest extends LocalFileMetadataTestCase {
-
- private GridTable table;
- private GTInfo info;
- private CompareTupleFilter timeComp0;
- private CompareTupleFilter timeComp1;
- private CompareTupleFilter timeComp2;
- private CompareTupleFilter timeComp3;
- private CompareTupleFilter timeComp4;
- private CompareTupleFilter timeComp5;
- private CompareTupleFilter timeComp6;
- private CompareTupleFilter ageComp1;
- private CompareTupleFilter ageComp2;
- private CompareTupleFilter ageComp3;
- private CompareTupleFilter ageComp4;
-
- @After
- public void after() throws Exception {
-
- this.cleanupTestMetadata();
- }
-
- @Before
- public void setup() throws IOException {
-
- this.createTestMetadata();
-
- table = newTestTable();
- info = table.getInfo();
-
- timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14"));
- timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
- timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
- timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
- timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15"));
- timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14"));
- ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
- ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
- ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
- ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
-
- }
-
- @Test
- public void verifySegmentSkipping() {
-
- ByteArray segmentStart = enc(info, 0, "2015-01-14");
- ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free
- ByteArray segmentEnd = enc(info, 0, "2015-01-15");
- assertEquals(segmentStart, segmentStartX);
-
- {
- LogicalTupleFilter filter = and(timeComp0, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());//scan range are [close,close]
- assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
- assertEquals(1, r.get(0).fuzzyKeys.size());
- assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
- {
- LogicalTupleFilter filter = and(timeComp2, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = and(timeComp4, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = and(timeComp5, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
- assertEquals("[[null, 10, null, null, null], [1421193600000, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
- {
- LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
- assertEquals(0, r.get(0).fuzzyKeys.size());
- }
- {
- //skip FALSE filter
- LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
- {
- //TRUE or FALSE filter
- LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[null, null]-[null, null]", r.get(0).toString());
- }
- {
- //TRUE or other filter
- LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[null, null]-[null, null]", r.get(0).toString());
- }
- }
-
- @Test
- public void verifySegmentSkipping2() {
- ByteArray segmentEnd = enc(info, 0, "2015-01-15");
-
- {
- LogicalTupleFilter filter = and(timeComp0, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());//scan range are [close,close]
- assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
- assertEquals(1, r.get(0).fuzzyKeys.size());
- assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
-
- {
- LogicalTupleFilter filter = and(timeComp5, ageComp1);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());//scan range are [close,close]
- }
- }
-
- @Test
- public void verifyScanRangePlanner() {
-
- // flatten or-and & hbase fuzzy value
- {
- LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
- assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
- assertEquals("[[null, 10, null, null, null], [null, 20, null, null, null]]", r.get(0).fuzzyKeys.toString());
- }
-
- // pre-evaluate ever false
- {
- LogicalTupleFilter filter = and(timeComp1, timeComp2);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(0, r.size());
- }
-
- // pre-evaluate ever true
- {
- LogicalTupleFilter filter = or(timeComp1, ageComp4);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals("[[null, null]-[null, null]]", r.toString());
- }
-
- // merge overlap range
- {
- LogicalTupleFilter filter = or(timeComp1, timeComp3);
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals("[[null, null]-[null, null]]", r.toString());
- }
-
- // merge too many ranges
- {
- LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
- CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, null, filter);
- List<GTScanRange> r = planner.planScanRanges();
- assertEquals(3, r.size());
- assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
- assertEquals("[1421280000000, 20]-[1421280000000, 20]", r.get(1).toString());
- assertEquals("[1421280000000, 30]-[1421280000000, 30]", r.get(2).toString());
- planner.setMaxScanRanges(2);
- List<GTScanRange> r2 = planner.planScanRanges();
- assertEquals("[[1421280000000, 10]-[1421280000000, 30]]", r2.toString());
- }
- }
-
- @Test
- public void verifyFirstRow() throws IOException {
- doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", //
- "[1421193600000, 30, Luke, 10, 10.5]", //
- "[1421280000000, 20, Dong, 10, 10.5]", //
- "[1421280000000, 20, Jason, 10, 10.5]", //
- "[1421280000000, 30, Xu, 10, 10.5]", //
- "[1421366400000, 20, Mahone, 10, 10.5]", //
- "[1421366400000, 20, Qianhao, 10, 10.5]", //
- "[1421366400000, 30, George, 10, 10.5]", //
- "[1421366400000, 30, Shaofeng, 10, 10.5]", //
- "[1421452800000, 10, Kejia, 10, 10.5]");
- }
-
- //for testing GTScanRequest serialization and deserialization
- public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
- ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
- GTScanRequest.serializer.serialize(origin, buffer);
- buffer.flip();
- GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer);
-
- Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs());
- Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01);
- return sGTScanRequest;
- }
-
- @Test
- public void verifyScanWithUnevaluatableFilter() throws IOException {
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1));
- LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1)));
- LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable);
-
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[]{"sum"}).setFilterPushDown(filter).createGTScanRequest();
-
- // note the unEvaluatable column 1 in filter is added to group by
- assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
-
- doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]");
- }
-
- @Test
- public void verifyScanWithEvaluatableFilter() throws IOException {
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[]{"sum"}).setFilterPushDown(filter).createGTScanRequest();
- // note the evaluatable column 1 in filter is added to returned columns but not in group by
- assertEquals("GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
-
- doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]");
- }
-
- @Test
- public void testFilterScannerPerf() throws IOException {
- GridTable table = newTestPerfTable();
- GTInfo info = table.getInfo();
-
- CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10"));
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- FilterResultCache.ENABLED = false;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = true;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = false;
- testFilterScannerPerfInner(table, info, filter);
- FilterResultCache.ENABLED = true;
- testFilterScannerPerfInner(table, info, filter);
- }
-
- @SuppressWarnings("unused")
- private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException {
- long start = System.currentTimeMillis();
- GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(filter).createGTScanRequest();
- IGTScanner scanner = table.scan(req);
- int i = 0;
- for (GTRecord r : scanner) {
- i++;
- }
- scanner.close();
- long end = System.currentTimeMillis();
- System.out.println((end - start) + "ms with filter cache enabled=" + FilterResultCache.ENABLED + ", " + i + " rows");
- }
-
- @Test
- public void verifyConvertFilterConstants1() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants2() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1<"9" round up to $1<"10"
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants3() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1<="9" round down to FALSE
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString());
- }
-
- @Test
- public void verifyConvertFilterConstants4() {
- GTInfo info = table.getInfo();
-
- TableDesc extTable = TableDesc.mockup("ext");
- TblColRef extColA = ColumnDesc.mockup(extTable, 1, "A", "timestamp").getRef();
- TblColRef extColB = ColumnDesc.mockup(extTable, 2, "B", "integer").getRef();
-
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
- List<TblColRef> colMapping = Lists.newArrayList();
- colMapping.add(extColA);
- colMapping.add(extColB);
-
- // $1 in ("9", "10", "15") has only "10" left
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString());
- }
-
- private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
- System.out.println(req);
- IGTScanner scanner = table.scan(req);
- int i = 0;
- for (GTRecord r : scanner) {
- System.out.println(r);
- if (verifyRows == null || i >= verifyRows.length) {
- Assert.fail();
- }
- assertEquals(verifyRows[i], r.toString());
- i++;
- }
- scanner.close();
- }
-
- public static ByteArray enc(GTInfo info, int col, String value) {
- ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
- info.codeSystem.encodeColumnValue(col, value, buf);
- return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
- }
-
- public static ExtractTupleFilter unevaluatable(TblColRef col) {
- ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
- r.addChild(new ColumnTupleFilter(col));
- return r;
- }
-
- public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
- CompareTupleFilter result = new CompareTupleFilter(op);
- result.addChild(new ColumnTupleFilter(col));
- result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
- return result;
- }
-
- public static LogicalTupleFilter and(TupleFilter... children) {
- return logic(FilterOperatorEnum.AND, children);
- }
-
- public static LogicalTupleFilter or(TupleFilter... children) {
- return logic(FilterOperatorEnum.OR, children);
- }
-
- public static LogicalTupleFilter not(TupleFilter child) {
- return logic(FilterOperatorEnum.NOT, child);
- }
-
- public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
- LogicalTupleFilter result = new LogicalTupleFilter(op);
- for (TupleFilter c : children) {
- result.addChild(c);
- }
- return result;
- }
-
- public static GridTable newTestTable() throws IOException {
- GTInfo info = newInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder = table.rebuild();
-
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
- builder.close();
-
- return table;
- }
-
- static GridTable newTestPerfTable() throws IOException {
- GTInfo info = newInfo();
- GTSimpleMemStore store = new GTSimpleMemStore(info);
- GridTable table = new GridTable(info, store);
-
- GTRecord r = new GTRecord(table.getInfo());
- GTBuilder builder = table.rebuild();
-
- for (int i = 0; i < 100000; i++) {
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
-
- for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
- }
- builder.close();
-
- return table;
- }
-
- static GTInfo newInfo() {
- Builder builder = GTInfo.builder();
- builder.setCodeSystem(newDictCodeSystem());
- builder.setColumns( //
- DataType.getType("timestamp"), //
- DataType.getType("integer"), //
- DataType.getType("varchar(10)"), //
- DataType.getType("bigint"), //
- DataType.getType("decimal") //
- );
- builder.setPrimaryKey(setOf(0, 1));
- builder.setColumnPreferIndex(setOf(0));
- builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
- builder.enableRowBlock(4);
- GTInfo info = builder.build();
- return info;
- }
-
- @SuppressWarnings("unchecked")
- private static CubeCodeSystem newDictCodeSystem() {
- DimensionEncoding[] dimEncs = new DimensionEncoding[3];
- dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
- dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString());
- return new CubeCodeSystem(dimEncs);
- }
-
- @SuppressWarnings("rawtypes")
- private static Dictionary newDictionaryOfString() {
- TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
- builder.addValue("Dong");
- builder.addValue("George");
- builder.addValue("Jason");
- builder.addValue("Kejia");
- builder.addValue("Luke");
- builder.addValue("Mahone");
- builder.addValue("Qianhao");
- builder.addValue("Shaofeng");
- builder.addValue("Xu");
- builder.addValue("Yang");
- return builder.build(0);
- }
-
- @SuppressWarnings("rawtypes")
- private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
- builder.addValue("10");
- builder.addValue("20");
- builder.addValue("30");
- builder.addValue("40");
- builder.addValue("50");
- builder.addValue("60");
- builder.addValue("70");
- builder.addValue("80");
- builder.addValue("90");
- builder.addValue("100");
- return builder.build(0);
- }
-
- public static ImmutableBitSet setOf(int... values) {
- BitSet set = new BitSet();
- for (int i : values)
- set.set(i);
- return new ImmutableBitSet(set);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
new file mode 100644
index 0000000..9f505f3
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.FuzzyValueCombination;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.gridtable.CubeGridTable;
+import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.gridtable.RecordComparators;
+import org.apache.kylin.cube.gridtable.ScanRangePlannerBase;
+import org.apache.kylin.cube.gridtable.SegmentGTStartAndEnd;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRange;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanRequestBuilder;
+import org.apache.kylin.gridtable.GTUtil;
+import org.apache.kylin.gridtable.IGTComparator;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CubeScanRangePlanner extends ScanRangePlannerBase {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
+
+ protected int maxScanRanges;
+ protected int maxFuzzyKeys;
+
+ //non-GT
+ protected CubeSegment cubeSegment;
+ protected CubeDesc cubeDesc;
+ protected Cuboid cuboid;
+
+ protected StorageContext context;
+
+ public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
+ Collection<FunctionDesc> metrics, StorageContext context) {
+ this.context = context;
+
+ this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+ this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
+ this.cubeSegment = cubeSegment;
+ this.cubeDesc = cubeSegment.getCubeDesc();
+ this.cuboid = cuboid;
+
+ Set<TblColRef> filterDims = Sets.newHashSet();
+ TupleFilter.collectColumns(filter, filterDims);
+
+ this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
+ CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
+
+ IGTComparator comp = gtInfo.getCodeSystem().getComparator();
+ //start key GTRecord compare to start key GTRecord
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
+ //stop key GTRecord compare to stop key GTRecord
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
+ //start key GTRecord compare to stop key GTRecord
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
+
+ //replace the constant values in filter to dictionary codes
+ this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
+
+ this.gtDimensions = mapping.makeGridTableColumns(dimensions);
+ this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
+ this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
+ this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
+
+ if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
+ int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
+ if (index >= 0) {
+ SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
+ this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
+ this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
+ this.gtPartitionCol = gtInfo.colRef(index);
+ }
+ }
+
+ }
+
+ /**
+ * constrcut GTScanRangePlanner with incomplete information. only be used for UT
+ * @param info
+ * @param gtStartAndEnd
+ * @param gtPartitionCol
+ * @param gtFilter
+ */
+ public CubeScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
+
+ this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+ this.maxFuzzyKeys = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
+ this.gtInfo = info;
+
+ IGTComparator comp = gtInfo.getCodeSystem().getComparator();
+ //start key GTRecord compare to start key GTRecord
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
+ //stop key GTRecord compare to stop key GTRecord
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
+ //start key GTRecord compare to stop key GTRecord
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
+
+ this.gtFilter = gtFilter;
+ this.gtStartAndEnd = gtStartAndEnd;
+ this.gtPartitionCol = gtPartitionCol;
+ }
+
+ public GTScanRequest planScanRequest() {
+ GTScanRequest scanRequest;
+ List<GTScanRange> scanRanges = this.planScanRanges();
+ if (scanRanges != null && scanRanges.size() != 0) {
+ GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
+ setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).//
+ setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).//
+ setStorageScanRowNumThreshold(context.getThreshold());
+
+ if (cubeDesc.supportsLimitPushDown()) {
+ builder.setStoragePushDownLimit(context.getStoragePushDownLimit());
+ }
+ scanRequest = builder.createGTScanRequest();
+ } else {
+ scanRequest = null;
+ }
+ return scanRequest;
+ }
+
+ /**
+ * Overwrite this method to provide smarter storage visit plans
+ * @return
+ */
+ public List<GTScanRange> planScanRanges() {
+ TupleFilter flatFilter = flattenToOrAndFilter(gtFilter);
+
+ List<Collection<ColumnRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter);
+
+ List<GTScanRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
+ for (Collection<ColumnRange> andDimRanges : orAndDimRanges) {
+ GTScanRange scanRange = newScanRange(andDimRanges);
+ if (scanRange != null)
+ scanRanges.add(scanRange);
+ }
+
+ List<GTScanRange> mergedRanges = mergeOverlapRanges(scanRanges);
+ mergedRanges = mergeTooManyRanges(mergedRanges, maxScanRanges);
+
+ return mergedRanges;
+ }
+
+ private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
+ Set<TblColRef> ret = Sets.newHashSet();
+ for (TblColRef col : input) {
+ if (cubeDesc.hasHostColumn(col)) {
+ for (TblColRef host : cubeDesc.getHostInfo(col).columns) {
+ ret.add(host);
+ }
+ } else {
+ ret.add(col);
+ }
+ }
+ return ret;
+ }
+
+ protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
+ GTRecord pkStart = new GTRecord(gtInfo);
+ GTRecord pkEnd = new GTRecord(gtInfo);
+ Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
+
+ List<GTRecord> fuzzyKeys;
+
+ for (ColumnRange range : andDimRanges) {
+ if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
+ int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
+ int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
+
+ if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
+ } else {
+ logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
+ gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
+ return null;
+ }
+ }
+
+ int col = range.column.getColumnDesc().getZeroBasedIndex();
+ if (!gtInfo.getPrimaryKey().get(col))
+ continue;
+
+ pkStart.set(col, range.begin);
+ pkEnd.set(col, range.end);
+
+ if (range.valueSet != null && !range.valueSet.isEmpty()) {
+ fuzzyValues.put(col, range.valueSet);
+ }
+ }
+
+ fuzzyKeys =
+
+ buildFuzzyKeys(fuzzyValues);
+ return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
+ }
+
+ private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
+ ArrayList<GTRecord> result = Lists.newArrayList();
+
+ if (fuzzyValueSet.isEmpty())
+ return result;
+
+ // debug/profiling purpose
+ if (BackdoorToggles.getDisableFuzzyKey()) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+
+ List<Map<Integer, ByteArray>> fuzzyValueCombinations = FuzzyValueCombination.calculate(fuzzyValueSet, maxFuzzyKeys);
+
+ for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
+
+ // BitSet bitSet = new BitSet(gtInfo.getColumnCount());
+ // for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
+ // bitSet.set(entry.getKey());
+ // }
+ GTRecord fuzzy = new GTRecord(gtInfo);
+ for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
+ fuzzy.set(entry.getKey(), entry.getValue());
+ }
+
+ result.add(fuzzy);
+ }
+ return result;
+ }
+
+ protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
+ if (ranges.size() <= 1) {
+ return ranges;
+ }
+
+ // sort ranges by start key
+ Collections.sort(ranges, new Comparator<GTScanRange>() {
+ @Override
+ public int compare(GTScanRange a, GTScanRange b) {
+ return rangeStartComparator.compare(a.pkStart, b.pkStart);
+ }
+ });
+
+ // merge the overlap range
+ List<GTScanRange> mergedRanges = new ArrayList<GTScanRange>();
+ int mergeBeginIndex = 0;
+ GTRecord mergeEnd = ranges.get(0).pkEnd;
+ for (int index = 1; index < ranges.size(); index++) {
+ GTScanRange range = ranges.get(index);
+
+ // if overlap, swallow it
+ if (rangeStartEndComparator.compare(range.pkStart, mergeEnd) <= 0) {
+ mergeEnd = rangeEndComparator.max(mergeEnd, range.pkEnd);
+ continue;
+ }
+
+ // not overlap, split here
+ GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, index));
+ mergedRanges.add(mergedRange);
+
+ // start new split
+ mergeBeginIndex = index;
+ mergeEnd = range.pkEnd;
+ }
+
+ // don't miss the last range
+ GTScanRange mergedRange = mergeKeyRange(ranges.subList(mergeBeginIndex, ranges.size()));
+ mergedRanges.add(mergedRange);
+
+ return mergedRanges;
+ }
+
+ private GTScanRange mergeKeyRange(List<GTScanRange> ranges) {
+ GTScanRange first = ranges.get(0);
+ if (ranges.size() == 1)
+ return first;
+
+ GTRecord start = first.pkStart;
+ GTRecord end = first.pkEnd;
+ List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
+
+ boolean hasNonFuzzyRange = false;
+ for (GTScanRange range : ranges) {
+ hasNonFuzzyRange = hasNonFuzzyRange || range.fuzzyKeys.isEmpty();
+ newFuzzyKeys.addAll(range.fuzzyKeys);
+ end = rangeEndComparator.max(end, range.pkEnd);
+ }
+
+ // if any range is non-fuzzy, then all fuzzy keys must be cleared
+ // also too many fuzzy keys will slow down HBase scan
+ if (hasNonFuzzyRange || newFuzzyKeys.size() > maxFuzzyKeys) {
+ newFuzzyKeys.clear();
+ }
+
+ return new GTScanRange(start, end, newFuzzyKeys);
+ }
+
+ protected List<GTScanRange> mergeTooManyRanges(List<GTScanRange> ranges, int maxRanges) {
+ if (ranges.size() <= maxRanges) {
+ return ranges;
+ }
+
+ // TODO: check the distance between range and merge the large distance range
+ List<GTScanRange> result = new ArrayList<GTScanRange>(1);
+ GTScanRange mergedRange = mergeKeyRange(ranges);
+ result.add(mergedRange);
+ return result;
+ }
+
+ public int getMaxScanRanges() {
+ return maxScanRanges;
+ }
+
+ public void setMaxScanRanges(int maxScanRanges) {
+ this.maxScanRanges = maxScanRanges;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 3b9d9c6..f32831a 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -23,10 +23,8 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -67,23 +65,13 @@ public class CubeSegmentScanner implements IGTScanner {
ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
- String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner();
CubeScanRangePlanner scanRangePlanner;
try {
- scanRangePlanner = (CubeScanRangePlanner) Class.forName(plannerName).getConstructor(CubeSegment.class, Cuboid.class, TupleFilter.class, Set.class, Set.class, Collection.class).newInstance(cubeSeg, cuboid, filter, dimensions, groups, metrics);
+ scanRangePlanner = new CubeScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics, context);
} catch (Exception e) {
throw new RuntimeException(e);
}
scanRequest = scanRangePlanner.planScanRequest();
- if (scanRequest != null) {
- scanRequest.setAllowStorageAggregation(context.isNeedStorageAggregation());
- scanRequest.setAggCacheMemThreshold(cubeSeg.getCubeInstance().getConfig().getQueryCoprocessorMemGB());
- scanRequest.setStorageScanRowNumThreshold(context.getThreshold());//TODO: devide by shard number?
-
- if (cubeSeg.getCubeDesc().supportsLimitPushDown()) {
- scanRequest.setStoragePushDownLimit(context.getStoragePushDownLimit());
- }
- }
scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e38557b4/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 86346f8..f0c2494 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.gtrecord;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -72,10 +73,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-
+
//cope with queries with no aggregations
RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
-
+
// Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
@@ -112,9 +113,9 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
TupleFilter filterD = translateDerived(filter, groupsD);
- context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD, exactAggregation));
- enableStoragePushDownLimit(cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context);
- setThreshold(dimensionsD, metrics, context); // set cautious threshold to prevent out of memory
+ context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD));
+ enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, context);
+ setThresholdIfNecessary(dimensionsD, metrics, context); // set cautious threshold to prevent out of memory
List<CubeSegmentScanner> scanners = Lists.newArrayList();
for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
@@ -229,9 +230,22 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return resultD;
}
- public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD, boolean isExactAggregation) {
- logger.info("Set isNeedStorageAggregation to " + !isExactAggregation);
- return !isExactAggregation;
+ public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) {
+
+ logger.info("GroupD :" + groupD);
+ logger.info("SingleValueD :" + singleValueD);
+ logger.info("Cuboid columns :" + cuboid.getColumns());
+
+ HashSet<TblColRef> temp = Sets.newHashSet();
+ temp.addAll(groupD);
+ temp.addAll(singleValueD);
+ if (cuboid.getColumns().size() == temp.size()) {
+ logger.info("Does not need storage aggregation");
+ return false;
+ } else {
+ logger.info("Need storage aggregation");
+ return true;
+ }
}
//exact aggregation was introduced back when we had some measures (like holistic distinct count) that is sensitive
@@ -268,7 +282,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
if (exact) {
- logger.info("exactAggregation is true");
+ logger.info("exactAggregation is true, cuboid id is " + cuboid.getId());
}
return exact;
}
@@ -355,7 +369,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
- private void setThreshold(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, StorageContext context) {
+ private void setThresholdIfNecessary(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, StorageContext context) {
boolean hasMemHungryMeasure = false;
for (FunctionDesc func : metrics) {
hasMemHungryMeasure |= func.getMeasureType().isMemoryHungry();
@@ -381,7 +395,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
- private void enableStoragePushDownLimit(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) {
+ private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter, Collection<FunctionDesc> functionDescs, StorageContext context) {
boolean possible = true;
boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
[16/43] kylin git commit: KYLIN-1698-BIGINT
Posted by sh...@apache.org.
KYLIN-1698-BIGINT
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3a1767f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3a1767f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3a1767f
Branch: refs/heads/v1.5.4-release2
Commit: e3a1767f07dcb7365480d0fa495083d8bfdd0344
Parents: f12a5e7
Author: chenzhx <34...@qq.com>
Authored: Tue Sep 6 17:20:15 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 15:48:50 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelEdit.js | 25 +++++++++++++++++++-
.../modelDesigner/conditions_settings.html | 10 ++++----
2 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a1767f/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index a916aad..2532fb4 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar");
+ return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint";
});
return columns;
};
@@ -69,6 +69,26 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
return type;
};
+ $scope.isBigInt=false;
+ $scope.partitionChange = function (dateColumn) {
+ var column = _.filter($scope.getColumnsByTable($scope.modelsManager.selectedModel.fact_table),function(_column){
+ var columnName=$scope.modelsManager.selectedModel.fact_table+"."+_column.name;
+ if(dateColumn==columnName)
+ return _column;
+ });
+ if(column[0].datatype==="bigint"){
+ $scope.isBigInt=true;
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_format=null;;
+ $scope.partitionColumn.hasSeparateTimeColumn=false;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
+ }
+ else{
+ $scope.isBigInt=false;
+ }
+
+ }
+
// ~ Define data
$scope.state = {
"modelSchema": "",
@@ -87,6 +107,9 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
ModelDescService.query({model_name: modelName}, function (model) {
if (model) {
modelsManager.selectedModel = model;
+ if(!$scope.modelsManager.selectedModel.partition_desc.partition_data_format){
+ $scope.isBigInt = true;
+ }
if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
$scope.partitionColumn.hasSeparateTimeColumn = true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a1767f/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index a4e49a9..693241c 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -46,7 +46,7 @@
<select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
- ng-if="state.mode=='edit'"
+ ng-if="state.mode=='edit'" ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonColumns(modelsManager.selectedModel.fact_table)" >
<option value="">--Select Partition Column--</option>
@@ -60,14 +60,14 @@
</div>
<!--Date Format-->
- <div class="form-group">
+ <div class="form-group" ng-if="isBigInt==false">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
<div class="col-xs-12 col-sm-6">
<select style="width: 100%" chosen
ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-if="state.mode=='edit'"
+ ng-if="state.mode=='edit'" ng-disable="isBigInt(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
<option value="">--Select Date Format--</option>
@@ -78,7 +78,7 @@
</div>
<!--Date Format-->
- <div class="form-group middle-popover">
+ <div class="form-group middle-popover" ng-if="isBigInt==false">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has a separate "time of the day" column ?</b> <i kylinpopover placement="right" title="Separate Time Column" template="separateTimeColumnTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6">
@@ -164,7 +164,7 @@
<script type="text/ng-template" id="partitionTip.html">
<ol>
<li>Partition date column not required,leave as default if cube always need full build</Li>
- <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR, etc.)</li>
+ <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR,BigInt, etc.)</li>
</ol>
</script>
[33/43] kylin git commit: KYLIN-1922 imporve CI
Posted by sh...@apache.org.
KYLIN-1922 imporve CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/942406bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/942406bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/942406bd
Branch: refs/heads/v1.5.4-release2
Commit: 942406bda3ec7405a6d2be27ba11bb38b5f88298
Parents: 466cf1a
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 23:01:00 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 23:01:00 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITKylinQueryTest.java | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/942406bd/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5f6af7a..3411c91 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -44,6 +44,8 @@ import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -119,6 +121,22 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
+ thrown.expect(SQLException.class);
+
+ //should not break at table duplicate check, should fail at model duplicate check
+ thrown.expectCause(new BaseMatcher<Throwable>() {
+ @Override
+ public boolean matches(Object item) {
+ if (item instanceof GTScanSelfTerminatedException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ }
+ });
try {
@@ -133,10 +151,6 @@ public class ITKylinQueryTest extends KylinTestBase {
RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
- } catch (SQLException e) {
- if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
- throw new RuntimeException();
- }
} finally {
//these two cubes has RAW measure, will disturb limit push down
[18/43] kylin git commit: KYLIN-1998 release job engine lock at
shutdown
Posted by sh...@apache.org.
KYLIN-1998 release job engine lock at shutdown
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/97778788
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/97778788
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/97778788
Branch: refs/heads/v1.5.4-release2
Commit: 97778788a0fea2bb4db4254264d42c58bdb8f7cc
Parents: 413bc9f
Author: Li Yang <li...@apache.org>
Authored: Wed Sep 7 17:37:57 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Sep 7 17:38:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/97778788/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 2dc1ab5..1ea3be0 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -49,6 +49,7 @@ import com.google.common.collect.Maps;
*/
public class DefaultScheduler implements Scheduler<AbstractExecutable>, ConnectionStateListener {
+ private JobLock jobLock;
private ExecutableManager executableManager;
private FetcherRunner fetcher;
private ScheduledExecutorService fetcherPool;
@@ -181,6 +182,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, final JobLock jobLock) throws SchedulerException {
+ this.jobLock = jobLock;
+
String serverMode = jobEngineConfig.getConfig().getServerMode();
if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) {
logger.info("server mode: " + serverMode + ", no need to run job scheduler");
@@ -216,6 +219,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
@Override
public void shutdown() throws SchedulerException {
+ logger.info("Shutingdown Job Engine ....");
+ jobLock.unlock();
fetcherPool.shutdown();
jobPool.shutdown();
}
[23/43] kylin git commit: KYLIN-2003 error start time
Posted by sh...@apache.org.
KYLIN-2003 error start time
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ded3b585
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ded3b585
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ded3b585
Branch: refs/heads/v1.5.4-release2
Commit: ded3b58583e76ab1234ac48459c264196c98f04b
Parents: d680169
Author: Jason <ji...@163.com>
Authored: Thu Sep 8 17:47:26 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu Sep 8 17:47:26 2016 +0800
----------------------------------------------------------------------
webapp/app/js/filters/filter.js | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ded3b585/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index 4e8d210..f9f7165 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -152,16 +152,16 @@ KylinApp
var convertedMillis = item;
if (gmttimezone.indexOf("GMT+") != -1) {
var offset = gmttimezone.substr(4, 1);
- convertedMillis = item + offset * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() + offset * 60 * 60000 + localOffset * 60000;
}
else if (gmttimezone.indexOf("GMT-") != -1) {
var offset = gmttimezone.substr(4, 1);
- convertedMillis = item - offset * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() - offset * 60 * 60000 + localOffset * 60000;
}
else {
// return PST by default
timezone = "PST";
- convertedMillis = item - 8 * 60 * 60000 + localOffset * 60000;
+ convertedMillis = new Date(item).getTime() - 8 * 60 * 60000 + localOffset * 60000;
}
return $filter('date')(convertedMillis, format) + " " + timezone;
[30/43] kylin git commit: KYLIN-2005 Move all storage side behavior
hints to GTScanRequest
Posted by sh...@apache.org.
KYLIN-2005 Move all storage side behavior hints to GTScanRequest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2c875d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2c875d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2c875d8
Branch: refs/heads/v1.5.4-release2
Commit: a2c875d8a2d06f23dd6467bbcc459bff82918295
Parents: e38557b
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 16:46:22 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 17:47:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/gridtable/GTScanRequest.java | 33 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 30 +-
.../kylin/gridtable/StorageSideBehavior.java | 30 +
.../apache/kylin/query/ITKylinQueryTest.java | 4 +-
.../common/coprocessor/CoprocessorBehavior.java | 30 -
.../observer/AggregateRegionObserver.java | 10 +-
.../observer/AggregationScanner.java | 16 +-
.../coprocessor/observer/ObserverEnabler.java | 6 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 88 +--
.../hbase/cube/v2/ExpectedSizeIterator.java | 4 +-
.../coprocessor/endpoint/CubeVisitService.java | 18 +-
.../endpoint/generated/CubeVisitProtos.java | 754 ++++---------------
.../endpoint/protobuf/CubeVisit.proto | 13 +-
.../observer/AggregateRegionObserverTest.java | 6 +-
14 files changed, 332 insertions(+), 710 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5d27028..3e57e86 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -59,6 +59,9 @@ public class GTScanRequest {
private String[] aggrMetricsFuncs;//
// hint to storage behavior
+ private String storageBehavior;
+ private long startTime;
+ private long timeout;
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
private int storageScanRowNumThreshold;
@@ -69,7 +72,7 @@ public class GTScanRequest {
GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, boolean allowStorageAggregation, //
- double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit) {
+ double aggCacheMemThreshold, int storageScanRowNumThreshold, int storagePushDownLimit, String storageBehavior, long startTime, long timeout) {
this.info = info;
if (ranges == null) {
this.ranges = Lists.newArrayList(new GTScanRange(new GTRecord(info), new GTRecord(info)));
@@ -83,6 +86,9 @@ public class GTScanRequest {
this.aggrMetrics = aggrMetrics;
this.aggrMetricsFuncs = aggrMetricsFuncs;
+ this.storageBehavior = storageBehavior;
+ this.startTime = startTime;
+ this.timeout = timeout;
this.allowStorageAggregation = allowStorageAggregation;
this.aggCacheMemThreshold = aggCacheMemThreshold;
this.storageScanRowNumThreshold = storageScanRowNumThreshold;
@@ -115,6 +121,10 @@ public class GTScanRequest {
}
}
+ public void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
private void validateFilterPushDown(GTInfo info) {
if (!hasFilterPushDown())
return;
@@ -280,6 +290,18 @@ public class GTScanRequest {
return this.storagePushDownLimit;
}
+ public String getStorageBehavior() {
+ return storageBehavior;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
@@ -320,6 +342,9 @@ public class GTScanRequest {
out.putDouble(value.aggCacheMemThreshold);
BytesUtil.writeVInt(value.storageScanRowNumThreshold, out);
BytesUtil.writeVInt(value.storagePushDownLimit, out);
+ BytesUtil.writeVLong(value.startTime, out);
+ BytesUtil.writeVLong(value.timeout, out);
+ BytesUtil.writeUTFString(value.storageBehavior, out);
}
@Override
@@ -350,11 +375,15 @@ public class GTScanRequest {
double sAggrCacheGB = in.getDouble();
int storageScanRowNumThreshold = BytesUtil.readVInt(in);
int storagePushDownLimit = BytesUtil.readVInt(in);
+ long startTime = BytesUtil.readVLong(in);
+ long timeout = BytesUtil.readVLong(in);
+ String storageBehavior = BytesUtil.readUTFString(in);
return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns).//
setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs).//
setFilterPushDown(sGTFilter).setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB).//
- setStorageScanRowNumThreshold(storageScanRowNumThreshold).setStoragePushDownLimit(storagePushDownLimit).createGTScanRequest();
+ setStorageScanRowNumThreshold(storageScanRowNumThreshold).setStoragePushDownLimit(storagePushDownLimit).//
+ setStartTime(startTime).setTimeout(timeout).setStorageBehavior(storageBehavior).createGTScanRequest();
}
private void serializeGTRecord(GTRecord gtRecord, ByteBuffer out) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index c4390cd..f542de1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -21,6 +21,7 @@ package org.apache.kylin.gridtable;
import java.util.BitSet;
import java.util.List;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -36,6 +37,9 @@ public class GTScanRequestBuilder {
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.
private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced.
+ private long startTime = -1;
+ private long timeout = -1;
+ private String storageBehavior = null;
public GTScanRequestBuilder setInfo(GTInfo info) {
this.info = info;
@@ -92,6 +96,21 @@ public class GTScanRequestBuilder {
return this;
}
+ public GTScanRequestBuilder setStartTime(long startTime) {
+ this.startTime = startTime;
+ return this;
+ }
+
+ public GTScanRequestBuilder setTimeout(long timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ public GTScanRequestBuilder setStorageBehavior(String storageBehavior) {
+ this.storageBehavior = storageBehavior;
+ return this;
+ }
+
public GTScanRequest createGTScanRequest() {
if (aggrGroupBy == null) {
aggrGroupBy = new ImmutableBitSet(new BitSet());
@@ -104,7 +123,14 @@ public class GTScanRequestBuilder {
if (aggrMetricsFuncs == null) {
aggrMetricsFuncs = new String[0];
}
-
- return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit);
+
+ if (storageBehavior == null) {
+ storageBehavior = BackdoorToggles.getCoprocessorBehavior() == null ? StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
+ }
+
+ this.startTime = startTime == -1 ? System.currentTimeMillis() : startTime;
+ this.timeout = timeout == -1 ? 300000 : timeout;
+
+ return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit, storageBehavior, startTime, timeout);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
new file mode 100644
index 0000000..7fa93e7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+/**
+ */
+public enum StorageSideBehavior {
+ RAW_SCAN, //on use RegionScanner to scan raw data, for testing hbase scan speed
+ SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
+ SCAN_FILTER_AGGR, //aggregate the result. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index fc2fd52..0efea64 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -40,7 +40,7 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
@@ -140,7 +140,7 @@ public class ITKylinQueryTest extends KylinTestBase {
});
Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
BackdoorToggles.setToggles(toggles);
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
deleted file mode 100644
index 5f21351..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.common.coprocessor;
-
-/**
- */
-public enum CoprocessorBehavior {
- RAW_SCAN, //on use RegionScanner to scan raw data, for testing hbase scan speed
- SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
- SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
- SCAN_FILTER_AGGR, //aggregate the result. Will return results
- SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
- SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY, // on each scan operation, delay for 10s to simulate slow queries, for test use
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index c7b650a..7139ca7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -85,15 +85,15 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
- CoprocessorBehavior coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ StorageSideBehavior storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
try {
byte[] behavior = scan.getAttribute(BEHAVIOR);
if (behavior != null && behavior.length != 0) {
- coprocessorBehavior = CoprocessorBehavior.valueOf(new String(behavior));
+ storageSideBehavior = StorageSideBehavior.valueOf(new String(behavior));
}
} catch (Exception e) {
LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
- coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
}
// start/end region operation & sync on scanner is suggested by the
@@ -103,7 +103,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner, coprocessorBehavior);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner, storageSideBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index be26142..a77f988 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -39,9 +39,9 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
- private CoprocessorBehavior behavior;
+ private StorageSideBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, CoprocessorBehavior behavior) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, StorageSideBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
@@ -79,23 +79,23 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (behavior == CoprocessorBehavior.SCAN) {
+ if (behavior == StorageSideBehavior.SCAN) {
//touch every byte of the cell so that the cost of scanning will be trully reflected
int endIndex = cell.getRowOffset() + cell.getRowLength();
for (int i = cell.getRowOffset(); i < endIndex; ++i) {
meaninglessByte += cell.getRowArray()[i];
}
} else {
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER.ordinal()) {
if (filter != null && filter.evaluate(tuple) == false)
continue;
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR.ordinal()) {
AggrKey aggKey = projector.getAggrKey(results);
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, results);
- if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
aggCache.checkMemoryUsage();
}
}
@@ -103,7 +103,7 @@ public class AggregationScanner implements RegionScanner {
}
}
- if (behavior == CoprocessorBehavior.SCAN) {
+ if (behavior == StorageSideBehavior.SCAN) {
System.out.println("meaningless byte is now " + meaninglessByte);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index f0e9bed..394b3e2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -35,7 +35,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -75,14 +75,14 @@ public class ObserverEnabler {
if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
return new ResultScannerAdapter(aggrScanner);
} else {
// debug/profiling purpose
String toggle = BackdoorToggles.getCoprocessorBehavior();
if (toggle == null) {
- toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ toggle = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5b48351..573951b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -47,7 +46,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
@@ -104,10 +102,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@Override
public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException {
- final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
-
- logger.info("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle);
-
Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard();
short shardNum = shardNumAndBaseShard.getFirst();
short cuboidBaseShard = shardNumAndBaseShard.getSecond();
@@ -130,39 +124,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
//TODO: raw scan can be constructed at region side to reduce traffic
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
- int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
- while (true) {
- try {
- ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize);
- BytesUtil.writeVInt(rawScans.size(), rawScanBuffer);
- for (RawScan rs : rawScans) {
- RawScan.serializer.serialize(rs, rawScanBuffer);
- }
- rawScanBuffer.flip();
- rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit());
- break;
- } catch (BufferOverflowException boe) {
- logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize);
- rawScanBufferSize *= 4;
- }
- }
+ rawScanByteString = serializeRawScans(rawScans);
+
scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
-
- int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
- while (true) {
- try {
- ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize);
- GTScanRequest.serializer.serialize(scanRequest, buffer);
- buffer.flip();
- scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit());
- break;
- } catch (BufferOverflowException boe) {
- logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize);
- scanRequestBufferSize *= 4;
- }
- }
-
- logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
+ final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
+ scanRequest.setTimeout(epResultItr.getRpcTimeout());
+ scanRequestByteString = serializeGTScanReq(scanRequest);
+
+ logger.info("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size());
for (RawScan rs : rawScans) {
@@ -172,7 +141,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size());
final AtomicLong totalScannedCount = new AtomicLong(0);
- final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
// KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries
// for different cubes until redeployment of coprocessor jar.
@@ -184,9 +152,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
- builder.setBehavior(toggle);
- builder.setStartTime(System.currentTimeMillis());
- builder.setTimeout(epResultItr.getRpcTimeout());
builder.setKylinProperties(kylinConfig.getConfigAsString());
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
@@ -260,6 +225,45 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return new GTBlobScatter(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
}
+ private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
+ ByteString scanRequestByteString;
+ int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
+ while (true) {
+ try {
+ ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize);
+ GTScanRequest.serializer.serialize(scanRequest, buffer);
+ buffer.flip();
+ scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit());
+ break;
+ } catch (BufferOverflowException boe) {
+ logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize);
+ scanRequestBufferSize *= 4;
+ }
+ }
+ return scanRequestByteString;
+ }
+
+ private ByteString serializeRawScans(List<RawScan> rawScans) {
+ ByteString rawScanByteString;
+ int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
+ while (true) {
+ try {
+ ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize);
+ BytesUtil.writeVInt(rawScans.size(), rawScanBuffer);
+ for (RawScan rs : rawScans) {
+ RawScan.serializer.serialize(rs, rawScanBuffer);
+ }
+ rawScanBuffer.flip();
+ rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit());
+ break;
+ } catch (BufferOverflowException boe) {
+ logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize);
+ rawScanBufferSize *= 4;
+ }
+ }
+ return rawScanByteString;
+ }
+
private String getStatsString(byte[] region, CubeVisitResponse result) {
StringBuilder sb = new StringBuilder();
Stats stats = result.getStats();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 442963f..f4729a3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -92,8 +92,8 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
if (coprocException instanceof GTScanSelfTerminatedException)
throw (GTScanSelfTerminatedException) coprocException;
else
- throw new RuntimeException("Error in coprocessor",coprocException);
-
+ throw new RuntimeException("Error in coprocessor", coprocException);
+
} else if (ret == null) {
throw new RuntimeException("Timeout visiting cube! Check why coprocessor exception is not sent back? In coprocessor Self-termination is checked every " + //
GTScanRequest.terminateCheckInterval + " scanned rows, the configured timeout(" + timeout + ") cannot support this many scans?");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 064d100..36adca1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -54,11 +54,11 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanTimeoutException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.BufferedMeasureEncoder;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.CellListIterator;
import org.apache.kylin.storage.hbase.cube.v2.CubeHBaseRPC;
import org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore;
@@ -198,10 +198,10 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
for (IntList intList : request.getHbaseColumnsToGTList()) {
hbaseColumnsToGT.add(intList.getIntsList());
}
- CoprocessorBehavior behavior = CoprocessorBehavior.valueOf(request.getBehavior());
+ StorageSideBehavior behavior = StorageSideBehavior.valueOf(scanReq.getStorageBehavior());
final List<RawScan> hbaseRawScans = deserializeRawScans(ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan())));
- appendProfileInfo(sb, "start latency: " + (this.serviceStartTime - request.getStartTime()));
+ appendProfileInfo(sb, "start latency: " + (this.serviceStartTime - scanReq.getStartTime()));
MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
@Override
@@ -228,7 +228,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
final Iterator<List<Cell>> allCellLists = Iterators.concat(cellListsForeachRawScan.iterator());
- if (behavior.ordinal() < CoprocessorBehavior.SCAN.ordinal()) {
+ if (behavior.ordinal() < StorageSideBehavior.SCAN.ordinal()) {
//this is only for CoprocessorBehavior.RAW_SCAN case to profile hbase scan speed
List<Cell> temp = Lists.newArrayList();
int counter = 0;
@@ -240,12 +240,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
appendProfileInfo(sb, "scanned " + counter);
}
- if (behavior.ordinal() < CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ if (behavior.ordinal() < StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
scanReq.disableAggCacheMemCheck(); // disable mem check if so told
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
- final long deadline = request.getTimeout() + this.serviceStartTime;
+ final long deadline = scanReq.getTimeout() + this.serviceStartTime;
final long storagePushDownLimit = scanReq.getStoragePushDownLimit();
final CellListIterator cellListIterator = new CellListIterator() {
@@ -285,12 +285,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
};
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, //
- request.getRowkeyPreambleSize(), CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(request.getBehavior()));
+ request.getRowkeyPreambleSize(), StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString().equals(scanReq.getStorageBehavior()));
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, //
- behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal(), //
- behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal(), deadline);
+ behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER.ordinal(), //
+ behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR.ordinal(), deadline);
ByteBuffer buffer = ByteBuffer.allocate(BufferedMeasureEncoder.DEFAULT_BUFFER_SIZE);
[31/43] kylin git commit: minor, remove unnecessary raw measures
Posted by sh...@apache.org.
minor, remove unnecessary raw measures
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/618cf28c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/618cf28c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/618cf28c
Branch: refs/heads/v1.5.4-release2
Commit: 618cf28c96d6b267d13ff737d2b3c550fc67e176
Parents: a2c875d
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 18:57:01 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 18:57:01 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/topn/TopNMeasureType.java | 2 +
.../test_case_data/localmeta/cube_desc/ssb.json | 409 +++++++------
.../test_kylin_cube_with_slr_desc.json | 389 +++++++-----
...st_kylin_cube_with_view_inner_join_desc.json | 388 +++++++-----
...est_kylin_cube_with_view_left_join_desc.json | 388 +++++++-----
.../test_kylin_cube_without_slr_desc.json | 58 +-
...t_kylin_cube_without_slr_left_join_desc.json | 587 +++++++++++--------
.../test_streaming_table_cube_desc.json | 245 ++++----
8 files changed, 1423 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index ed22d61..0756056 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -274,6 +274,8 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (sum.isSum() == false)
return false;
+ if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0)
+ return false;
TblColRef sumCol = sum.getParameter().getColRefs().get(0);
return sumCol.equals(topnNumCol);
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/ssb.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb.json b/examples/test_case_data/localmeta/cube_desc/ssb.json
index d3ea10b..4903979 100644
--- a/examples/test_case_data/localmeta/cube_desc/ssb.json
+++ b/examples/test_case_data/localmeta/cube_desc/ssb.json
@@ -1,179 +1,256 @@
{
- "uuid" : "5c44df30-daec-486e-af90-927bf7851057",
- "name" : "ssb",
- "description" : "",
- "dimensions" : [ {
- "name" : "SSB.PART_DERIVED",
- "table" : "SSB.PART",
- "column" : null,
- "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
- }, {
- "name" : "C_CITY",
- "table" : "SSB.CUSTOMER",
- "column" : "C_CITY",
- "derived" : null
- }, {
- "name" : "C_REGION",
- "table" : "SSB.CUSTOMER",
- "column" : "C_REGION",
- "derived" : null
- }, {
- "name" : "C_NATION",
- "table" : "SSB.CUSTOMER",
- "column" : "C_NATION",
- "derived" : null
- }, {
- "name" : "S_CITY",
- "table" : "SSB.SUPPLIER",
- "column" : "S_CITY",
- "derived" : null
- }, {
- "name" : "S_REGION",
- "table" : "SSB.SUPPLIER",
- "column" : "S_REGION",
- "derived" : null
- }, {
- "name" : "S_NATION",
- "table" : "SSB.SUPPLIER",
- "column" : "S_NATION",
- "derived" : null
- }, {
- "name" : "D_YEAR",
- "table" : "SSB.DATES",
- "column" : "D_YEAR",
- "derived" : null
- }, {
- "name" : "D_YEARMONTH",
- "table" : "SSB.DATES",
- "column" : "D_YEARMONTH",
- "derived" : null
- }, {
- "name" : "D_YEARMONTHNUM",
- "table" : "SSB.DATES",
- "column" : "D_YEARMONTHNUM",
- "derived" : null
- }, {
- "name" : "D_WEEKNUMINYEAR",
- "table" : "SSB.DATES",
- "column" : "D_WEEKNUMINYEAR",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "_COUNT_",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ "uuid": "5c44df30-daec-486e-af90-927bf7851057",
+ "name": "ssb",
+ "description": "",
+ "dimensions": [
+ {
+ "name": "SSB.PART_DERIVED",
+ "table": "SSB.PART",
+ "column": null,
+ "derived": [
+ "P_MFGR",
+ "P_CATEGORY",
+ "P_BRAND"
+ ]
+ },
+ {
+ "name": "C_CITY",
+ "table": "SSB.CUSTOMER",
+ "column": "C_CITY",
+ "derived": null
+ },
+ {
+ "name": "C_REGION",
+ "table": "SSB.CUSTOMER",
+ "column": "C_REGION",
+ "derived": null
+ },
+ {
+ "name": "C_NATION",
+ "table": "SSB.CUSTOMER",
+ "column": "C_NATION",
+ "derived": null
+ },
+ {
+ "name": "S_CITY",
+ "table": "SSB.SUPPLIER",
+ "column": "S_CITY",
+ "derived": null
+ },
+ {
+ "name": "S_REGION",
+ "table": "SSB.SUPPLIER",
+ "column": "S_REGION",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_REVENUE",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "LO_REVENUE",
- "next_parameter" : null
+ {
+ "name": "S_NATION",
+ "table": "SSB.SUPPLIER",
+ "column": "S_NATION",
+ "derived": null
+ },
+ {
+ "name": "D_YEAR",
+ "table": "SSB.DATES",
+ "column": "D_YEAR",
+ "derived": null
+ },
+ {
+ "name": "D_YEARMONTH",
+ "table": "SSB.DATES",
+ "column": "D_YEARMONTH",
+ "derived": null
+ },
+ {
+ "name": "D_YEARMONTHNUM",
+ "table": "SSB.DATES",
+ "column": "D_YEARMONTHNUM",
+ "derived": null
+ },
+ {
+ "name": "D_WEEKNUMINYEAR",
+ "table": "SSB.DATES",
+ "column": "D_WEEKNUMINYEAR",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "_COUNT_",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_SUPPLYCOST",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "LO_SUPPLYCOST",
- "next_parameter" : null
+ {
+ "name": "TOTAL_REVENUE",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "LO_REVENUE",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOTAL_V_REVENUE",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "V_REVENUE",
- "next_parameter" : null
+ {
+ "name": "TOTAL_SUPPLYCOST",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "LO_SUPPLYCOST",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "LO_PARTKEY",
- "encoding" : "dict"
- }, {
- "column" : "C_CITY",
- "encoding" : "dict"
- }, {
- "column" : "C_REGION",
- "encoding" : "dict"
- }, {
- "column" : "C_NATION",
- "encoding" : "dict"
- }, {
- "column" : "S_CITY",
- "encoding" : "dict"
- }, {
- "column" : "S_REGION",
- "encoding" : "dict"
- }, {
- "column" : "S_NATION",
- "encoding" : "dict"
- }, {
- "column" : "D_YEAR",
- "encoding" : "dict"
- }, {
- "column" : "D_YEARMONTH",
- "encoding" : "dict"
- }, {
- "column" : "D_YEARMONTHNUM",
- "encoding" : "dict"
- }, {
- "column" : "D_WEEKNUMINYEAR",
- "encoding" : "dict"
- } ]
+ {
+ "name": "TOTAL_V_REVENUE",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "V_REVENUE",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "LO_PARTKEY",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_CITY",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_REGION",
+ "encoding": "dict"
+ },
+ {
+ "column": "C_NATION",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_CITY",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_REGION",
+ "encoding": "dict"
+ },
+ {
+ "column": "S_NATION",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEAR",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEARMONTH",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_YEARMONTHNUM",
+ "encoding": "dict"
+ },
+ {
+ "column": "D_WEEKNUMINYEAR",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : "5iV8LVYs+PmVUju8QNQ5TQ==",
- "last_modified" : 1457503036686,
- "model_name" : "ssb",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
- } ]
- } ]
+ "signature": "5iV8LVYs+PmVUju8QNQ5TQ==",
+ "last_modified": 1457503036686,
+ "model_name": "ssb",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "_COUNT_",
+ "TOTAL_REVENUE",
+ "TOTAL_SUPPLYCOST",
+ "TOTAL_V_REVENUE"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
- "mandatory_dims" : [ "D_YEAR" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "LO_PARTKEY",
+ "C_CITY",
+ "C_REGION",
+ "C_NATION",
+ "S_CITY",
+ "S_REGION",
+ "S_NATION",
+ "D_YEAR",
+ "D_YEARMONTH",
+ "D_YEARMONTHNUM",
+ "D_WEEKNUMINYEAR"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "C_REGION",
+ "C_NATION",
+ "C_CITY"
+ ],
+ [
+ "S_REGION",
+ "S_NATION",
+ "S_CITY"
+ ],
+ [
+ "D_YEARMONTH",
+ "D_YEARMONTHNUM",
+ "D_WEEKNUMINYEAR"
+ ]
+ ],
+ "mandatory_dims": [
+ "D_YEAR"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : [ ],
- "status_need_notify" : [ ],
- "partition_date_start" : 694224000000,
- "partition_date_end" : 3153600000000,
- "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "override_kylin_properties" : {
- "kylin.hbase.default.compression.codec" : "lz4",
- "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ ],
+ "notify_list": [],
+ "status_need_notify": [],
+ "partition_date_start": 694224000000,
+ "partition_date_end": 3153600000000,
+ "auto_merge_time_ranges": [
+ 604800000,
+ 2419200000
+ ],
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
+ "override_kylin_properties": {
+ "kylin.hbase.default.compression.codec": "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid": "true"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 4064fcb..f62d196 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -1,172 +1,245 @@
{
- "uuid" : "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
-
- "name" : "test_kylin_cube_with_slr_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- }, {
- "name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
+ "name": "test_kylin_cube_with_slr_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ },
+ {
+ "name": "SELLER_ID",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "SELLER_ID",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "seller_id",
- "encoding" : "int:4",
- "isShardBy" : true
- }, {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "fixed_length:18"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "fixed_length:12"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "seller_id",
+ "encoding": "int:4",
+ "isShardBy": true
+ },
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "fixed_length:18"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "fixed_length:12"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801271,
- "model_name" : "test_kylin_inner_join_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801271,
+ "model_name": "test_kylin_inner_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "seller_id", "slr_segment_cd" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : ["seller_id"],
- "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" ] ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name",
+ "seller_id",
+ "slr_segment_cd"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "seller_id"
+ ],
+ "joint_dims": [
+ [
+ "lstg_format_name",
+ "lstg_site_id",
+ "slr_segment_cd"
+ ]
+ ]
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "partition_date_start" : 0
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
+ "partition_date_start": 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
index d4c64b5..e3a3e70 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
@@ -1,169 +1,249 @@
{
- "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_inner_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_with_view_inner_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.V_TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_inner_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_inner_join_view_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
index 0388c0e..b17fbff 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
@@ -1,169 +1,249 @@
{
- "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_left_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_with_view_left_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.V_TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "decimal(19,4)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_left_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_left_join_view_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 28328e4..7de2ae2 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -1,5 +1,4 @@
{
-
"uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
"name": "test_kylin_cube_without_slr_desc",
"description": null,
@@ -160,54 +159,19 @@
"returntype": "extendedcolumn(100)"
},
"dependent_measure_ref": null
- }, {
- "name" : "CAL_DT_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "CAL_DT",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "LSTG_FORMAT_NAME_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "LEAF_CATEG_ID_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LEAF_CATEG_ID",
- "next_parameter" : null
- },
- "returntype" : "raw"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "PRICE_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ },
+ {
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
},
- "returntype" : "raw"
+ "returntype": "raw"
},
- "dependent_measure_ref" : null
+ "dependent_measure_ref": null
}
],
"rowkey": {
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index ca1b35c..4270aab 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -1,293 +1,360 @@
{
- "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_without_slr_left_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name": "test_kylin_cube_without_slr_left_join_desc",
+ "description": null,
+ "dimensions": [
+ {
+ "name": "CAL_DT",
+ "table": "EDW.TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ {
+ "name": "CATEGORY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "UPD_DATE",
+ "UPD_USER"
+ ]
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEGORY_HIERARCHY",
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+ },
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "DEFAULT.TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "SITE_ID",
+ "table": "EDW.TEST_SITES",
+ "column": "{FK}",
+ "derived": [
+ "SITE_NAME",
+ "CRE_USER"
+ ]
+ },
+ {
+ "name": "SELLER_TYPE_CD",
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "column": "{FK}",
+ "derived": [
+ "SELLER_TYPE_DESC"
+ ]
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_CNT_BITMAP",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
+ {
+ "name": "GMV_MIN",
+ "function": {
+ "expression": "MIN",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bitmap"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SITE_NAME_BITMAP",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SITE_NAME",
- "next_parameter" : null
+ {
+ "name": "GMV_MAX",
+ "function": {
+ "expression": "MAX",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,4)"
},
- "returntype" : "bitmap"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "hllc(10)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "TOP_SELLER",
- "function" : {
- "expression" : "TOP_N",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "topn(100)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "CAL_DT_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "CAL_DT",
- "next_parameter" : null
+ {
+ "name": "SELLER_CNT_BITMAP",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ },
+ "returntype": "bitmap"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "LSTG_FORMAT_NAME_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
+ {
+ "name": "SITE_NAME_BITMAP",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "SITE_NAME",
+ "next_parameter": null
+ },
+ "returntype": "bitmap"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "LEAF_CATEG_ID_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "LEAF_CATEG_ID",
- "next_parameter" : null
+ {
+ "name": "SELLER_FORMAT_CNT",
+ "function": {
+ "expression": "COUNT_DISTINCT",
+ "parameter": {
+ "type": "column",
+ "value": "LSTG_FORMAT_NAME",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ }
+ },
+ "returntype": "hllc(10)"
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "PRICE_RAW",
- "function" : {
- "expression" : "RAW",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
+ {
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ }
+ },
+ "returntype": "topn(100)",
+ "configuration": {
+ "topn.encoding.SELLER_ID": "int:4"
+ }
},
- "returntype" : "raw"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "dictionaries" : [
{
- "column" : "SITE_NAME",
+ "name": "PRICE_RAW",
+ "function": {
+ "expression": "RAW",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": null
+ },
+ "returntype": "raw"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "dictionaries": [
+ {
+ "column": "SITE_NAME",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
}
],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "cal_dt",
+ "encoding": "dict"
+ },
+ {
+ "column": "leaf_categ_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "meta_categ_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl2_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "categ_lvl3_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_format_name",
+ "encoding": "dict"
+ },
+ {
+ "column": "lstg_site_id",
+ "encoding": "dict"
+ },
+ {
+ "column": "slr_segment_cd",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_left_join_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum", "CAL_DT_RAW", "LSTG_FORMAT_NAME_RAW", "LEAF_CATEG_ID_RAW", "PRICE_RAW" ]
- } ]
- }, {
- "name" : "f2",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "seller_cnt_bitmap", "site_name_bitmap", "seller_format_cnt"]
- } ]
- }, {
- "name" : "f3",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "top_seller" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801311,
+ "model_name": "test_kylin_left_join_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "f1",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "gmv_sum",
+ "gmv_min",
+ "gmv_max",
+ "trans_cnt",
+ "item_count_sum",
+ "CAL_DT_RAW",
+ "LSTG_FORMAT_NAME_RAW",
+ "LEAF_CATEG_ID_RAW",
+ "PRICE_RAW"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "f2",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "seller_cnt_bitmap",
+ "site_name_bitmap",
+ "seller_format_cnt"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "f3",
+ "columns": [
+ {
+ "qualifier": "m",
+ "measure_refs": [
+ "top_seller"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "lstg_format_name",
+ "lstg_site_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": [
+ [
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ]
+ ]
+ }
+ },
+ {
+ "includes": [
+ "cal_dt",
+ "categ_lvl2_name",
+ "categ_lvl3_name",
+ "leaf_categ_id",
+ "meta_categ_name"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME"
+ ]
+ ],
+ "mandatory_dims": [
+ "cal_dt"
+ ],
+ "joint_dims": []
+ }
}
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ ],
+ "notify_list": null,
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"override_kylin_properties": {
"kylin.job.cubing.inmem.sampling.hll.precision": "16"
},
http://git-wip-us.apache.org/repos/asf/kylin/blob/618cf28c/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index ef10c1e..f2c4e72 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -1,118 +1,155 @@
{
- "uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd192",
-
- "name" : "test_streaming_table_cube_desc",
- "description" : "",
- "dimensions" : [ {
- "name" : "DEFAULT.STREAMING_TABLE.SITE",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "SITE",
- "derived" : null
- }, {
- "name" : "DEFAULT.STREAMING_TABLE.ITM",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "ITM",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "DAY_START",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "HOUR_START",
- "derived" : null
- }, {
- "name" : "TIME",
- "table" : "DEFAULT.STREAMING_TABLE",
- "column" : "MINUTE_START",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "_COUNT_",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
+ "uuid": "901ed15e-7769-4c66-b7ae-fbdc971cd192",
+ "name": "test_streaming_table_cube_desc",
+ "description": "",
+ "dimensions": [
+ {
+ "name": "DEFAULT.STREAMING_TABLE.SITE",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "SITE",
+ "derived": null
+ },
+ {
+ "name": "DEFAULT.STREAMING_TABLE.ITM",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "ITM",
+ "derived": null
+ },
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "DAY_START",
+ "derived": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "GMV",
- "next_parameter" : null
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "HOUR_START",
+ "derived": null
+ },
+ {
+ "name": "TIME",
+ "table": "DEFAULT.STREAMING_TABLE",
+ "column": "MINUTE_START",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "_COUNT_",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
},
- "returntype" : "decimal(19,6)"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "GMV",
+ "next_parameter": null
+ },
+ "returntype": "decimal(19,6)"
},
- "returntype" : "bigint"
+ "dependent_measure_ref": null
},
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "DAY_START",
- "encoding" : "dict"
- }, {
- "column" : "HOUR_START",
- "encoding" : "dict"
- }, {
- "column" : "MINUTE_START",
- "encoding" : "dict"
- }, {
- "column" : "SITE",
- "encoding" : "dict"
- }, {
- "column" : "ITM",
- "encoding" : "dict"
- } ]
+ {
+ "name": "ITEM_COUNT_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "ITEM_COUNT",
+ "next_parameter": null
+ },
+ "returntype": "bigint"
+ },
+ "dependent_measure_ref": null
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "DAY_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "HOUR_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "MINUTE_START",
+ "encoding": "dict"
+ },
+ {
+ "column": "SITE",
+ "encoding": "dict"
+ },
+ {
+ "column": "ITM",
+ "encoding": "dict"
+ }
+ ]
},
- "signature" : null,
- "last_modified" : 1448959801314,
- "model_name" : "test_streaming_table_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "_COUNT_", "GMV_SUM", "ITEM_COUNT_SUM" ]
- } ]
- } ]
+ "signature": null,
+ "last_modified": 1448959801314,
+ "model_name": "test_streaming_table_model_desc",
+ "null_string": null,
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "_COUNT_",
+ "GMV_SUM",
+ "ITEM_COUNT_SUM"
+ ]
+ }
+ ]
+ }
+ ]
},
- "aggregation_groups" : [ {
- "includes" : [ "DAY_START", "HOUR_START", "ITM", "MINUTE_START", "SITE" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "DAY_START", "HOUR_START", "MINUTE_START" ] ],
- "mandatory_dims" : [ ],
- "joint_dims" : [ ]
+ "aggregation_groups": [
+ {
+ "includes": [
+ "DAY_START",
+ "HOUR_START",
+ "ITM",
+ "MINUTE_START",
+ "SITE"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "DAY_START",
+ "HOUR_START",
+ "MINUTE_START"
+ ]
+ ],
+ "mandatory_dims": [],
+ "joint_dims": []
+ }
}
- } ],
+ ],
"override_kylin_properties": {
"kylin.cube.algorithm": "inmem"
},
- "notify_list" : [ ],
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
+ "notify_list": [],
+ "status_need_notify": [],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 2,
+ "storage_type": 2,
"partition_date_start": 0
}
\ No newline at end of file
[08/43] kylin git commit: KYLIN-1984 fix CI
Posted by sh...@apache.org.
KYLIN-1984 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9c8f40e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9c8f40e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9c8f40e
Branch: refs/heads/v1.5.4-release2
Commit: a9c8f40ee7423180d60313047b329d0c91d0dfb9
Parents: 99d0d75
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Sep 5 11:09:59 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Sep 5 11:09:59 2016 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9c8f40e/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
index 18d0a27..dd52f82 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
@@ -54,7 +54,7 @@ public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
}
private void verifyOverride(KylinConfig base, KylinConfig override) {
- assertEquals("", base.getHbaseDefaultCompressionCodec());
+ assertEquals("none", base.getHbaseDefaultCompressionCodec());
assertEquals("lz4", override.getHbaseDefaultCompressionCodec());
}
}
[42/43] kylin git commit: KYLIN-1983 add license header
Posted by sh...@apache.org.
KYLIN-1983 add license header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5c85017
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5c85017
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5c85017
Branch: refs/heads/v1.5.4-release2
Commit: c5c85017c135f03ac72ebbc9d1bb51f796eb0551
Parents: b941f11
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 09:58:48 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamDataLoader.java | 19 ++++++++++++++++++-
.../model/validation/rule/FunctionRuleTest.java | 18 ++++++++++++++++++
.../optrule/AggregateMultipleExpandRule.java | 18 ++++++++++++++++++
.../optrule/AggregateProjectReduceRule.java | 18 ++++++++++++++++++
.../PasswordPlaceHolderConfigurerTest.java | 18 ++++++++++++++++++
tomcat-ext/pom.xml | 18 ++++++++++++++++++
6 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
index 50fc883..2f7d54d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamDataLoader.java
@@ -1,6 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.job.streaming;
-import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
index 48e01e3..e041080 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.cube.model.validation.rule;
import org.apache.kylin.common.KylinConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
index eb7a03d..03a0674 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.query.optrule;
import com.google.common.base.Predicate;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
index 8c446e4..f6ac61a 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.query.optrule;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
index 3afd2ca..8f53084 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/security/PasswordPlaceHolderConfigurerTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
package org.apache.kylin.rest.security;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5c85017/tomcat-ext/pom.xml
----------------------------------------------------------------------
diff --git a/tomcat-ext/pom.xml b/tomcat-ext/pom.xml
index 1a171ee..f6af642 100644
--- a/tomcat-ext/pom.xml
+++ b/tomcat-ext/pom.xml
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
[29/43] kylin git commit: KYLIN-2005 Move all storage side behavior
hints to GTScanRequest
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index d9cef88..b0688b7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,115 +11,64 @@ public final class CubeVisitProtos {
public interface CubeVisitRequestOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // required string behavior = 1;
+ // required bytes gtScanRequest = 1;
/**
- * <code>required string behavior = 1;</code>
- */
- boolean hasBehavior();
- /**
- * <code>required string behavior = 1;</code>
- */
- java.lang.String getBehavior();
- /**
- * <code>required string behavior = 1;</code>
- */
- com.google.protobuf.ByteString
- getBehaviorBytes();
-
- // required bytes gtScanRequest = 2;
- /**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
boolean hasGtScanRequest();
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
com.google.protobuf.ByteString getGtScanRequest();
- // required bytes hbaseRawScan = 3;
+ // required bytes hbaseRawScan = 2;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
boolean hasHbaseRawScan();
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
com.google.protobuf.ByteString getHbaseRawScan();
- // required int32 rowkeyPreambleSize = 4;
+ // required int32 rowkeyPreambleSize = 3;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
boolean hasRowkeyPreambleSize();
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
int getRowkeyPreambleSize();
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>
getHbaseColumnsToGTList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index);
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
int getHbaseColumnsToGTCount();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index);
- // required int64 startTime = 6;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- boolean hasStartTime();
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- long getStartTime();
-
- // required int64 timeout = 7;
+ // required string kylinProperties = 5;
/**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- boolean hasTimeout();
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- long getTimeout();
-
- // required string kylinProperties = 8;
- /**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -127,7 +76,7 @@ public final class CubeVisitProtos {
*/
boolean hasKylinProperties();
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -135,7 +84,7 @@ public final class CubeVisitProtos {
*/
java.lang.String getKylinProperties();
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -197,44 +146,29 @@ public final class CubeVisitProtos {
}
case 10: {
bitField0_ |= 0x00000001;
- behavior_ = input.readBytes();
+ gtScanRequest_ = input.readBytes();
break;
}
case 18: {
bitField0_ |= 0x00000002;
- gtScanRequest_ = input.readBytes();
- break;
- }
- case 26: {
- bitField0_ |= 0x00000004;
hbaseRawScan_ = input.readBytes();
break;
}
- case 32: {
- bitField0_ |= 0x00000008;
+ case 24: {
+ bitField0_ |= 0x00000004;
rowkeyPreambleSize_ = input.readInt32();
break;
}
- case 42: {
- if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
+ case 34: {
+ if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>();
- mutable_bitField0_ |= 0x00000010;
+ mutable_bitField0_ |= 0x00000008;
}
hbaseColumnsToGT_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.PARSER, extensionRegistry));
break;
}
- case 48: {
- bitField0_ |= 0x00000010;
- startTime_ = input.readInt64();
- break;
- }
- case 56: {
- bitField0_ |= 0x00000020;
- timeout_ = input.readInt64();
- break;
- }
- case 66: {
- bitField0_ |= 0x00000040;
+ case 42: {
+ bitField0_ |= 0x00000008;
kylinProperties_ = input.readBytes();
break;
}
@@ -246,7 +180,7 @@ public final class CubeVisitProtos {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
- if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) {
+ if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
}
this.unknownFields = unknownFields.build();
@@ -785,196 +719,105 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required string behavior = 1;
- public static final int BEHAVIOR_FIELD_NUMBER = 1;
- private java.lang.Object behavior_;
- /**
- * <code>required string behavior = 1;</code>
- */
- public boolean hasBehavior() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public java.lang.String getBehavior() {
- java.lang.Object ref = behavior_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- behavior_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public com.google.protobuf.ByteString
- getBehaviorBytes() {
- java.lang.Object ref = behavior_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- behavior_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // required bytes gtScanRequest = 2;
- public static final int GTSCANREQUEST_FIELD_NUMBER = 2;
+ // required bytes gtScanRequest = 1;
+ public static final int GTSCANREQUEST_FIELD_NUMBER = 1;
private com.google.protobuf.ByteString gtScanRequest_;
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
- // required bytes hbaseRawScan = 3;
- public static final int HBASERAWSCAN_FIELD_NUMBER = 3;
+ // required bytes hbaseRawScan = 2;
+ public static final int HBASERAWSCAN_FIELD_NUMBER = 2;
private com.google.protobuf.ByteString hbaseRawScan_;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
- // required int32 rowkeyPreambleSize = 4;
- public static final int ROWKEYPREAMBLESIZE_FIELD_NUMBER = 4;
+ // required int32 rowkeyPreambleSize = 3;
+ public static final int ROWKEYPREAMBLESIZE_FIELD_NUMBER = 3;
private int rowkeyPreambleSize_;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public boolean hasRowkeyPreambleSize() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public int getRowkeyPreambleSize() {
return rowkeyPreambleSize_;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
- public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
+ public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
return hbaseColumnsToGT_.size();
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
return hbaseColumnsToGT_.get(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
return hbaseColumnsToGT_.get(index);
}
- // required int64 startTime = 6;
- public static final int STARTTIME_FIELD_NUMBER = 6;
- private long startTime_;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000010) == 0x00000010);
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public long getStartTime() {
- return startTime_;
- }
-
- // required int64 timeout = 7;
- public static final int TIMEOUT_FIELD_NUMBER = 7;
- private long timeout_;
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public boolean hasTimeout() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public long getTimeout() {
- return timeout_;
- }
-
- // required string kylinProperties = 8;
- public static final int KYLINPROPERTIES_FIELD_NUMBER = 8;
+ // required string kylinProperties = 5;
+ public static final int KYLINPROPERTIES_FIELD_NUMBER = 5;
private java.lang.Object kylinProperties_;
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public boolean hasKylinProperties() {
- return ((bitField0_ & 0x00000040) == 0x00000040);
+ return ((bitField0_ & 0x00000008) == 0x00000008);
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -995,7 +838,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -1016,13 +859,10 @@ public final class CubeVisitProtos {
}
private void initFields() {
- behavior_ = "";
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
rowkeyPreambleSize_ = 0;
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- startTime_ = 0L;
- timeout_ = 0L;
kylinProperties_ = "";
}
private byte memoizedIsInitialized = -1;
@@ -1030,10 +870,6 @@ public final class CubeVisitProtos {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
- if (!hasBehavior()) {
- memoizedIsInitialized = 0;
- return false;
- }
if (!hasGtScanRequest()) {
memoizedIsInitialized = 0;
return false;
@@ -1046,14 +882,6 @@ public final class CubeVisitProtos {
memoizedIsInitialized = 0;
return false;
}
- if (!hasStartTime()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasTimeout()) {
- memoizedIsInitialized = 0;
- return false;
- }
if (!hasKylinProperties()) {
memoizedIsInitialized = 0;
return false;
@@ -1066,28 +894,19 @@ public final class CubeVisitProtos {
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getBehaviorBytes());
+ output.writeBytes(1, gtScanRequest_);
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, gtScanRequest_);
+ output.writeBytes(2, hbaseRawScan_);
}
if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeBytes(3, hbaseRawScan_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- output.writeInt32(4, rowkeyPreambleSize_);
+ output.writeInt32(3, rowkeyPreambleSize_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
- output.writeMessage(5, hbaseColumnsToGT_.get(i));
- }
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- output.writeInt64(6, startTime_);
+ output.writeMessage(4, hbaseColumnsToGT_.get(i));
}
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- output.writeInt64(7, timeout_);
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
- output.writeBytes(8, getKylinPropertiesBytes());
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeBytes(5, getKylinPropertiesBytes());
}
getUnknownFields().writeTo(output);
}
@@ -1100,35 +919,23 @@ public final class CubeVisitProtos {
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getBehaviorBytes());
+ .computeBytesSize(1, gtScanRequest_);
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, gtScanRequest_);
+ .computeBytesSize(2, hbaseRawScan_);
}
if (((bitField0_ & 0x00000004) == 0x00000004)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(3, hbaseRawScan_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(4, rowkeyPreambleSize_);
+ .computeInt32Size(3, rowkeyPreambleSize_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(5, hbaseColumnsToGT_.get(i));
+ .computeMessageSize(4, hbaseColumnsToGT_.get(i));
}
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(6, startTime_);
- }
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(7, timeout_);
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(8, getKylinPropertiesBytes());
+ .computeBytesSize(5, getKylinPropertiesBytes());
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
@@ -1153,11 +960,6 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest) obj;
boolean result = true;
- result = result && (hasBehavior() == other.hasBehavior());
- if (hasBehavior()) {
- result = result && getBehavior()
- .equals(other.getBehavior());
- }
result = result && (hasGtScanRequest() == other.hasGtScanRequest());
if (hasGtScanRequest()) {
result = result && getGtScanRequest()
@@ -1175,16 +977,6 @@ public final class CubeVisitProtos {
}
result = result && getHbaseColumnsToGTList()
.equals(other.getHbaseColumnsToGTList());
- result = result && (hasStartTime() == other.hasStartTime());
- if (hasStartTime()) {
- result = result && (getStartTime()
- == other.getStartTime());
- }
- result = result && (hasTimeout() == other.hasTimeout());
- if (hasTimeout()) {
- result = result && (getTimeout()
- == other.getTimeout());
- }
result = result && (hasKylinProperties() == other.hasKylinProperties());
if (hasKylinProperties()) {
result = result && getKylinProperties()
@@ -1203,10 +995,6 @@ public final class CubeVisitProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasBehavior()) {
- hash = (37 * hash) + BEHAVIOR_FIELD_NUMBER;
- hash = (53 * hash) + getBehavior().hashCode();
- }
if (hasGtScanRequest()) {
hash = (37 * hash) + GTSCANREQUEST_FIELD_NUMBER;
hash = (53 * hash) + getGtScanRequest().hashCode();
@@ -1223,14 +1011,6 @@ public final class CubeVisitProtos {
hash = (37 * hash) + HBASECOLUMNSTOGT_FIELD_NUMBER;
hash = (53 * hash) + getHbaseColumnsToGTList().hashCode();
}
- if (hasStartTime()) {
- hash = (37 * hash) + STARTTIME_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getStartTime());
- }
- if (hasTimeout()) {
- hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getTimeout());
- }
if (hasKylinProperties()) {
hash = (37 * hash) + KYLINPROPERTIES_FIELD_NUMBER;
hash = (53 * hash) + getKylinProperties().hashCode();
@@ -1345,26 +1125,20 @@ public final class CubeVisitProtos {
public Builder clear() {
super.clear();
- behavior_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000002);
rowkeyPreambleSize_ = 0;
- bitField0_ = (bitField0_ & ~0x00000008);
+ bitField0_ = (bitField0_ & ~0x00000004);
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
hbaseColumnsToGTBuilder_.clear();
}
- startTime_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000020);
- timeout_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000040);
kylinProperties_ = "";
- bitField0_ = (bitField0_ & ~0x00000080);
+ bitField0_ = (bitField0_ & ~0x00000010);
return this;
}
@@ -1396,38 +1170,26 @@ public final class CubeVisitProtos {
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
to_bitField0_ |= 0x00000001;
}
- result.behavior_ = behavior_;
+ result.gtScanRequest_ = gtScanRequest_;
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000002;
}
- result.gtScanRequest_ = gtScanRequest_;
+ result.hbaseRawScan_ = hbaseRawScan_;
if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
to_bitField0_ |= 0x00000004;
}
- result.hbaseRawScan_ = hbaseRawScan_;
- if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
- to_bitField0_ |= 0x00000008;
- }
result.rowkeyPreambleSize_ = rowkeyPreambleSize_;
if (hbaseColumnsToGTBuilder_ == null) {
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
}
result.hbaseColumnsToGT_ = hbaseColumnsToGT_;
} else {
result.hbaseColumnsToGT_ = hbaseColumnsToGTBuilder_.build();
}
- if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
- to_bitField0_ |= 0x00000010;
- }
- result.startTime_ = startTime_;
- if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
- to_bitField0_ |= 0x00000020;
- }
- result.timeout_ = timeout_;
- if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
- to_bitField0_ |= 0x00000040;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000008;
}
result.kylinProperties_ = kylinProperties_;
result.bitField0_ = to_bitField0_;
@@ -1446,11 +1208,6 @@ public final class CubeVisitProtos {
public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other) {
if (other == org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.getDefaultInstance()) return this;
- if (other.hasBehavior()) {
- bitField0_ |= 0x00000001;
- behavior_ = other.behavior_;
- onChanged();
- }
if (other.hasGtScanRequest()) {
setGtScanRequest(other.getGtScanRequest());
}
@@ -1464,7 +1221,7 @@ public final class CubeVisitProtos {
if (!other.hbaseColumnsToGT_.isEmpty()) {
if (hbaseColumnsToGT_.isEmpty()) {
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
ensureHbaseColumnsToGTIsMutable();
hbaseColumnsToGT_.addAll(other.hbaseColumnsToGT_);
@@ -1477,7 +1234,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_.dispose();
hbaseColumnsToGTBuilder_ = null;
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
hbaseColumnsToGTBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getHbaseColumnsToGTFieldBuilder() : null;
@@ -1486,14 +1243,8 @@ public final class CubeVisitProtos {
}
}
}
- if (other.hasStartTime()) {
- setStartTime(other.getStartTime());
- }
- if (other.hasTimeout()) {
- setTimeout(other.getTimeout());
- }
if (other.hasKylinProperties()) {
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = other.kylinProperties_;
onChanged();
}
@@ -1502,10 +1253,6 @@ public final class CubeVisitProtos {
}
public final boolean isInitialized() {
- if (!hasBehavior()) {
-
- return false;
- }
if (!hasGtScanRequest()) {
return false;
@@ -1518,14 +1265,6 @@ public final class CubeVisitProtos {
return false;
}
- if (!hasStartTime()) {
-
- return false;
- }
- if (!hasTimeout()) {
-
- return false;
- }
if (!hasKylinProperties()) {
return false;
@@ -1552,192 +1291,118 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required string behavior = 1;
- private java.lang.Object behavior_ = "";
- /**
- * <code>required string behavior = 1;</code>
- */
- public boolean hasBehavior() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public java.lang.String getBehavior() {
- java.lang.Object ref = behavior_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- behavior_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public com.google.protobuf.ByteString
- getBehaviorBytes() {
- java.lang.Object ref = behavior_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- behavior_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder setBehavior(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- behavior_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder clearBehavior() {
- bitField0_ = (bitField0_ & ~0x00000001);
- behavior_ = getDefaultInstance().getBehavior();
- onChanged();
- return this;
- }
- /**
- * <code>required string behavior = 1;</code>
- */
- public Builder setBehaviorBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- behavior_ = value;
- onChanged();
- return this;
- }
-
- // required bytes gtScanRequest = 2;
+ // required bytes gtScanRequest = 1;
private com.google.protobuf.ByteString gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public Builder setGtScanRequest(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000001;
gtScanRequest_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes gtScanRequest = 2;</code>
+ * <code>required bytes gtScanRequest = 1;</code>
*/
public Builder clearGtScanRequest() {
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000001);
gtScanRequest_ = getDefaultInstance().getGtScanRequest();
onChanged();
return this;
}
- // required bytes hbaseRawScan = 3;
+ // required bytes hbaseRawScan = 2;
private com.google.protobuf.ByteString hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public Builder setHbaseRawScan(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000004;
+ bitField0_ |= 0x00000002;
hbaseRawScan_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes hbaseRawScan = 3;</code>
+ * <code>required bytes hbaseRawScan = 2;</code>
*/
public Builder clearHbaseRawScan() {
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000002);
hbaseRawScan_ = getDefaultInstance().getHbaseRawScan();
onChanged();
return this;
}
- // required int32 rowkeyPreambleSize = 4;
+ // required int32 rowkeyPreambleSize = 3;
private int rowkeyPreambleSize_ ;
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public boolean hasRowkeyPreambleSize() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public int getRowkeyPreambleSize() {
return rowkeyPreambleSize_;
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public Builder setRowkeyPreambleSize(int value) {
- bitField0_ |= 0x00000008;
+ bitField0_ |= 0x00000004;
rowkeyPreambleSize_ = value;
onChanged();
return this;
}
/**
- * <code>required int32 rowkeyPreambleSize = 4;</code>
+ * <code>required int32 rowkeyPreambleSize = 3;</code>
*/
public Builder clearRowkeyPreambleSize() {
- bitField0_ = (bitField0_ & ~0x00000008);
+ bitField0_ = (bitField0_ & ~0x00000004);
rowkeyPreambleSize_ = 0;
onChanged();
return this;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_ =
java.util.Collections.emptyList();
private void ensureHbaseColumnsToGTIsMutable() {
- if (!((bitField0_ & 0x00000010) == 0x00000010)) {
+ if (!((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>(hbaseColumnsToGT_);
- bitField0_ |= 0x00000010;
+ bitField0_ |= 0x00000008;
}
}
@@ -1745,7 +1410,7 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder> hbaseColumnsToGTBuilder_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1755,7 +1420,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1765,7 +1430,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1775,7 +1440,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1792,7 +1457,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1806,7 +1471,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1822,7 +1487,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1839,7 +1504,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1853,7 +1518,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1867,7 +1532,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addAllHbaseColumnsToGT(
java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> values) {
@@ -1881,12 +1546,12 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder clearHbaseColumnsToGT() {
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000010);
+ bitField0_ = (bitField0_ & ~0x00000008);
onChanged();
} else {
hbaseColumnsToGTBuilder_.clear();
@@ -1894,7 +1559,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder removeHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1907,14 +1572,14 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder getHbaseColumnsToGTBuilder(
int index) {
return getHbaseColumnsToGTFieldBuilder().getBuilder(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -1924,7 +1589,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
@@ -1935,14 +1600,14 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder() {
return getHbaseColumnsToGTFieldBuilder().addBuilder(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder(
int index) {
@@ -1950,7 +1615,7 @@ public final class CubeVisitProtos {
index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 5;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder>
getHbaseColumnsToGTBuilderList() {
@@ -1963,7 +1628,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>(
hbaseColumnsToGT_,
- ((bitField0_ & 0x00000010) == 0x00000010),
+ ((bitField0_ & 0x00000008) == 0x00000008),
getParentForChildren(),
isClean());
hbaseColumnsToGT_ = null;
@@ -1971,118 +1636,20 @@ public final class CubeVisitProtos {
return hbaseColumnsToGTBuilder_;
}
- // required int64 startTime = 6;
- private long startTime_ ;
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public long getStartTime() {
- return startTime_;
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public Builder setStartTime(long value) {
- bitField0_ |= 0x00000020;
- startTime_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 startTime = 6;</code>
- *
- * <pre>
- *when client start the request
- * </pre>
- */
- public Builder clearStartTime() {
- bitField0_ = (bitField0_ & ~0x00000020);
- startTime_ = 0L;
- onChanged();
- return this;
- }
-
- // required int64 timeout = 7;
- private long timeout_ ;
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public boolean hasTimeout() {
- return ((bitField0_ & 0x00000040) == 0x00000040);
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public long getTimeout() {
- return timeout_;
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public Builder setTimeout(long value) {
- bitField0_ |= 0x00000040;
- timeout_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 timeout = 7;</code>
- *
- * <pre>
- *how long client will wait
- * </pre>
- */
- public Builder clearTimeout() {
- bitField0_ = (bitField0_ & ~0x00000040);
- timeout_ = 0L;
- onChanged();
- return this;
- }
-
- // required string kylinProperties = 8;
+ // required string kylinProperties = 5;
private java.lang.Object kylinProperties_ = "";
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public boolean hasKylinProperties() {
- return ((bitField0_ & 0x00000080) == 0x00000080);
+ return ((bitField0_ & 0x00000010) == 0x00000010);
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2100,7 +1667,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2120,7 +1687,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2131,26 +1698,26 @@ public final class CubeVisitProtos {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = value;
onChanged();
return this;
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
* </pre>
*/
public Builder clearKylinProperties() {
- bitField0_ = (bitField0_ & ~0x00000080);
+ bitField0_ = (bitField0_ & ~0x00000010);
kylinProperties_ = getDefaultInstance().getKylinProperties();
onChanged();
return this;
}
/**
- * <code>required string kylinProperties = 8;</code>
+ * <code>required string kylinProperties = 5;</code>
*
* <pre>
* kylin properties
@@ -2161,7 +1728,7 @@ public final class CubeVisitProtos {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000080;
+ bitField0_ |= 0x00000010;
kylinProperties_ = value;
onChanged();
return this;
@@ -4521,26 +4088,25 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\370\001\n\020C" +
- "ubeVisitRequest\022\020\n\010behavior\030\001 \002(\t\022\025\n\rgtS" +
- "canRequest\030\002 \002(\014\022\024\n\014hbaseRawScan\030\003 \002(\014\022\032" +
- "\n\022rowkeyPreambleSize\030\004 \002(\005\0223\n\020hbaseColum" +
- "nsToGT\030\005 \003(\0132\031.CubeVisitRequest.IntList\022" +
- "\021\n\tstartTime\030\006 \002(\003\022\017\n\007timeout\030\007 \002(\003\022\027\n\017k" +
- "ylinProperties\030\010 \002(\t\032\027\n\007IntList\022\014\n\004ints\030" +
- "\001 \003(\005\"\321\002\n\021CubeVisitResponse\022\026\n\016compresse",
- "dRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitRe" +
- "sponse.Stats\032\372\001\n\005Stats\022\030\n\020serviceStartTi" +
- "me\030\001 \001(\003\022\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scan" +
- "nedRowCount\030\003 \001(\003\022\032\n\022aggregatedRowCount\030" +
- "\004 \001(\003\022\025\n\rsystemCpuLoad\030\005 \001(\001\022\036\n\026freePhys" +
- "icalMemorySize\030\006 \001(\001\022\031\n\021freeSwapSpaceSiz" +
- "e\030\007 \001(\001\022\020\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(" +
- "\t\022\026\n\016normalComplete\030\n \001(\0052F\n\020CubeVisitSe" +
- "rvice\0222\n\tvisitCube\022\021.CubeVisitRequest\032\022." +
- "CubeVisitResponseB`\nEorg.apache.kylin.st",
- "orage.hbase.cube.v2.coprocessor.endpoint" +
- ".generatedB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "/endpoint/protobuf/CubeVisit.proto\"\302\001\n\020C" +
+ "ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
+ "\n\014hbaseRawScan\030\002 \002(\014\022\032\n\022rowkeyPreambleSi" +
+ "ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
+ "eVisitRequest.IntList\022\027\n\017kylinProperties" +
+ "\030\005 \002(\t\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\321\002\n\021Cube" +
+ "VisitResponse\022\026\n\016compressedRows\030\001 \002(\014\022\'\n" +
+ "\005stats\030\002 \002(\0132\030.CubeVisitResponse.Stats\032\372",
+ "\001\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026\n\016se" +
+ "rviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 " +
+ "\001(\003\022\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\rsyste" +
+ "mCpuLoad\030\005 \001(\001\022\036\n\026freePhysicalMemorySize" +
+ "\030\006 \001(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020\n\010hos" +
+ "tname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016normalCom" +
+ "plete\030\n \001(\0052F\n\020CubeVisitService\0222\n\tvisit" +
+ "Cube\022\021.CubeVisitRequest\032\022.CubeVisitRespo" +
+ "nseB`\nEorg.apache.kylin.storage.hbase.cu" +
+ "be.v2.coprocessor.endpoint.generatedB\017Cu",
+ "beVisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4552,7 +4118,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "Behavior", "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "StartTime", "Timeout", "KylinProperties", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index e1de070..c84f4f3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -30,14 +30,11 @@ option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
message CubeVisitRequest {
- required string behavior = 1;
- required bytes gtScanRequest = 2;
- required bytes hbaseRawScan = 3;
- required int32 rowkeyPreambleSize = 4;
- repeated IntList hbaseColumnsToGT = 5;
- required int64 startTime = 6;//when client start the request
- required int64 timeout = 7;//how long client will wait
- required string kylinProperties = 8; // kylin properties
+ required bytes gtScanRequest = 1;
+ required bytes hbaseRawScan = 2;
+ required int32 rowkeyPreambleSize = 3;
+ repeated IntList hbaseColumnsToGT = 4;
+ required string kylinProperties = 5; // kylin properties
message IntList {
repeated int32 ints = 1;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2c875d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index f8e2644..390930a 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -40,7 +40,7 @@ import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -121,7 +121,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -170,7 +170,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[17/43] kylin git commit: KYLIN-1698-INT
Posted by sh...@apache.org.
KYLIN-1698-INT
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/413bc9f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/413bc9f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/413bc9f2
Branch: refs/heads/v1.5.4-release2
Commit: 413bc9f2d6c73c3c4822ba3ce0ab9da405cd5b9c
Parents: e3a1767
Author: chenzhx <34...@qq.com>
Authored: Wed Sep 7 13:48:21 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Sep 7 15:49:31 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelEdit.js | 7 +++++--
webapp/app/partials/modelDesigner/conditions_settings.html | 4 ++--
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/413bc9f2/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 2532fb4..65e17fa 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint";
+ return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint"||column.datatype==="int";
});
return columns;
};
@@ -71,12 +71,15 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.isBigInt=false;
$scope.partitionChange = function (dateColumn) {
+ if(dateColumn==null) {
+ return;
+ }
var column = _.filter($scope.getColumnsByTable($scope.modelsManager.selectedModel.fact_table),function(_column){
var columnName=$scope.modelsManager.selectedModel.fact_table+"."+_column.name;
if(dateColumn==columnName)
return _column;
});
- if(column[0].datatype==="bigint"){
+ if(column[0].datatype==="bigint"||column[0].datatype==="int"){
$scope.isBigInt=true;
$scope.modelsManager.selectedModel.partition_desc.partition_date_format=null;;
$scope.partitionColumn.hasSeparateTimeColumn=false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/413bc9f2/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 693241c..f0390e5 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -67,7 +67,7 @@
<select style="width: 100%" chosen
ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-if="state.mode=='edit'" ng-disable="isBigInt(modelsManager.selectedModel.partition_desc.partition_date_column)"
+ ng-if="state.mode=='edit'"
data-placement=""
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
<option value="">--Select Date Format--</option>
@@ -164,7 +164,7 @@
<script type="text/ng-template" id="partitionTip.html">
<ol>
<li>Partition date column not required,leave as default if cube always need full build</Li>
- <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR,BigInt, etc.)</li>
+ <li>Column should contain date value (type can be Date, Timestamp, String, VARCHAR, Int, BigInt, etc.)</li>
</ol>
</script>
[32/43] kylin git commit: KYLIN-1922 fix CI
Posted by sh...@apache.org.
KYLIN-1922 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/466cf1af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/466cf1af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/466cf1af
Branch: refs/heads/v1.5.4-release2
Commit: 466cf1afb19ee1acb4f99ff72ac61e7689617957
Parents: 618cf28
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 9 22:27:17 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Sep 9 22:27:17 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITKylinQueryTest.java | 53 ++++++++------------
1 file changed, 21 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/466cf1af/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 0efea64..5f6af7a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
+import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
@@ -40,12 +41,9 @@ import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -122,41 +120,32 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
- thrown.expect(SQLException.class);
-
- //should not break at table duplicate check, should fail at model duplicate check
- thrown.expectCause(new BaseMatcher<Throwable>() {
- @Override
- public boolean matches(Object item) {
- if (item instanceof GTScanSelfTerminatedException) {
- return true;
- }
- return false;
- }
-
- @Override
- public void describeTo(Description description) {
- }
- });
+ try {
- Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
- BackdoorToggles.setToggles(toggles);
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 10ms for every scan
+ BackdoorToggles.setToggles(toggles);
- KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "0.03");//set timeout to 9s
- //these two cubes has RAW measure, will disturb limit push down
- RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout", null, true);
+ } catch (SQLException e) {
+ if (!(e.getCause() instanceof GTScanSelfTerminatedException)) {
+ throw new RuntimeException();
+ }
+ } finally {
- //these two cubes has RAW measure, will disturb limit push down
- RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
- RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
+ //these two cubes has RAW measure, will disturb limit push down
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.remove("CUBE[name=test_kylin_cube_without_slr_inner_join_empty]");
- KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
- BackdoorToggles.cleanToggles();
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cube.visit.timeout.times", "1");//set timeout to 9s
+ BackdoorToggles.cleanToggles();
+ }
}
//don't try to ignore this test, try to clean your "temp" folder
[40/43] kylin git commit: minor, fix CI
Posted by sh...@apache.org.
minor, fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e95abde
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e95abde
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e95abde
Branch: refs/heads/v1.5.4-release2
Commit: 5e95abdec6da71a609cea80fa6569328b0c1e6cf
Parents: 3450c0f
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Sep 10 19:26:29 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Sep 10 19:26:42 2016 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e95abde/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 3411c91..de68c7a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -121,6 +121,11 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
+ if (HBaseStorage.overwriteStorageQuery != null) {
+ //v1 engine does not suit
+ return;
+ }
+
thrown.expect(SQLException.class);
//should not break at table duplicate check, should fail at model duplicate check
[04/43] kylin git commit: KYLIN-1767 td witdh update
Posted by sh...@apache.org.
KYLIN-1767 td witdh update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dae9778
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dae9778
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dae9778
Branch: refs/heads/v1.5.4-release2
Commit: 7dae9778c1f360261b465e38db47e954cf226590
Parents: 9e2970b
Author: Jason <ji...@163.com>
Authored: Fri Sep 2 19:56:47 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Sep 2 19:57:40 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/measures.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae9778/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 065b735..56c86be 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -251,7 +251,7 @@
<td>
<select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
ng-model="groupby_column.name"
- ng-options="column as column for column in getGroupBYColumns()" style="width:323px;">
+ ng-options="column as column for column in getGroupBYColumns()" style="width:200px;">
<option value="">--Select A Column--</option>
</select>
</td>
[12/43] kylin git commit: KYLIN-1962 use one file by default
Posted by sh...@apache.org.
KYLIN-1962 use one file by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae72c257
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae72c257
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae72c257
Branch: refs/heads/v1.5.4-release2
Commit: ae72c25705002f70df5996a4101acacbd6af7db6
Parents: 6c35c85
Author: shaofengshi <sh...@apache.org>
Authored: Tue Sep 6 10:40:24 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Sep 6 12:00:54 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 41 ++++++++++++++++++-
build/conf/kylin_account.properties | 42 --------------------
.../test_case_data/sandbox/kylin.properties | 5 +++
.../sandbox/kylin_account.properties | 13 ------
4 files changed, 44 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae72c257/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index c20488a..ed86bdb 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -84,9 +84,14 @@ kylin.job.run.as.remote.cmd=false
# Only necessary when kylin.job.run.as.remote.cmd=true
kylin.job.remote.cli.hostname=
-
kylin.job.remote.cli.port=22
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.username=
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.password=
+
# Used by test cases to prepare synthetic data for sample cube
kylin.job.remote.cli.working.dir=/tmp/kylin
@@ -146,11 +151,43 @@ kylin.query.cache.enabled=true
# with "testing" profile, user can use pre-defined name/pwd like KYLIN/ADMIN to login
kylin.security.profile=testing
+### SECURITY ###
+# Default roles and admin roles in LDAP, for ldap and saml
+acl.defaultRole=ROLE_ANALYST,ROLE_MODELER
+acl.adminRole=ROLE_ADMIN
+
+# LDAP authentication configuration
+ldap.server=ldap://ldap_server:389
+ldap.username=
+ldap.password=
+
+# LDAP user account directory;
+ldap.user.searchBase=
+ldap.user.searchPattern=
+ldap.user.groupSearchBase=
+
+# LDAP service account directory
+ldap.service.searchBase=
+ldap.service.searchPattern=
+ldap.service.groupSearchBase=
+
+## SAML configurations for SSO
+# SAML IDP metadata file location
+saml.metadata.file=classpath:sso_metadata.xml
+saml.metadata.entityBaseURL=https://hostname/kylin
+saml.context.scheme=https
+saml.context.serverName=hostname
+saml.context.serverPort=443
+saml.context.contextPath=/kylin
+
### MAIL ###
# If true, will send email notification;
mail.enabled=false
-
+mail.host=
+mail.username=
+mail.password=
+mail.sender=
### WEB ###
# Help info, format{name|displayName|link}, optional
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae72c257/build/conf/kylin_account.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin_account.properties b/build/conf/kylin_account.properties
deleted file mode 100644
index e98c142..0000000
--- a/build/conf/kylin_account.properties
+++ /dev/null
@@ -1,42 +0,0 @@
-### JOB ###
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-### SECURITY ###
-# Default roles and admin roles in LDAP, for ldap and saml
-acl.defaultRole=ROLE_ANALYST,ROLE_MODELER
-acl.adminRole=ROLE_ADMIN
-
-# LDAP authentication configuration
-ldap.server=ldap://ldap_server:389
-ldap.username=
-ldap.password=
-
-# LDAP user account directory;
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-
-# LDAP service account directory
-ldap.service.searchBase=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-
-# SAML configurations for SSO
-# SAML IDP metadata file location
-saml.metadata.file=classpath:sso_metadata.xml
-saml.metadata.entityBaseURL=https://hostname/kylin
-saml.context.scheme=https
-saml.context.serverName=hostname
-saml.context.serverPort=443
-saml.context.contextPath=/kylin
-
-### MAIL ###
-mail.host=
-mail.username=
-mail.password=
-mail.sender=
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae72c257/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index a6f89df..1d1d9ba 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -67,6 +67,11 @@ kylin.job.run.as.remote.cmd=false
# Only necessary when kylin.job.run.as.remote.cmd=true
kylin.job.remote.cli.hostname=sandbox
+kylin.job.remote.cli.username=root
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.password=hadoop
+
# Used by test cases to prepare synthetic data for sample cube
kylin.job.remote.cli.working.dir=/tmp/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae72c257/examples/test_case_data/sandbox/kylin_account.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_account.properties b/examples/test_case_data/sandbox/kylin_account.properties
deleted file mode 100644
index 0dfa5f7..0000000
--- a/examples/test_case_data/sandbox/kylin_account.properties
+++ /dev/null
@@ -1,13 +0,0 @@
-### JOB ###
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=root
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=hadoop
-
-### MAIL ###
-mail.host=
-mail.username=
-mail.password=
-mail.sender=
[43/43] kylin git commit: KYLIN-1983 only keep apache-license in main
pom.xml
Posted by sh...@apache.org.
KYLIN-1983 only keep apache-license in main pom.xml
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c59d63de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c59d63de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c59d63de
Branch: refs/heads/v1.5.4-release2
Commit: c59d63de6fc404456bd59f4ca5d36e48fb251637
Parents: c5c8501
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 11 10:00:54 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 11 10:13:28 2016 +0800
----------------------------------------------------------------------
kylin-it/pom.xml | 63 ---------------------------------------
pom.xml | 11 +++++++
query/pom.xml | 68 ------------------------------------------
storage-hbase/pom.xml | 73 ----------------------------------------------
4 files changed, 11 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f2e03b6..e14fa7d 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -332,69 +332,6 @@
</plugins>
</build>
</profile>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- test data -->
- <exclude>src/test/**/*.dat</exclude>
- <exclude>src/test/**/*.expected</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d5925b9..38f9365 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1215,6 +1215,17 @@
<exclude>**/*.DEF</exclude>
<exclude>**/*.isl</exclude>
<exclude>**/*.isproj</exclude>
+
+ <!-- protobuf generated -->
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
+ </exclude>
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+ </exclude>
+ <exclude>
+ src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+ </exclude>
</excludes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 0937b5a..8bb72ae 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -158,72 +158,4 @@
</exclusions>
</dependency>
</dependencies>
-
-
- <profiles>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- test data -->
- <exclude>src/test/**/*.dat</exclude>
- <exclude>src/test/**/*.expected</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c59d63de/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index b21a931..b332354 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -150,77 +150,4 @@
</plugins>
</build>
- <profiles>
- <profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <!-- protobuf generated -->
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
- </exclude>
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
- </exclude>
- <exclude>
- src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
- </exclude>
- <exclude>**/src/test/resources/**</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
</project>
[38/43] kylin git commit: KYLIN-2004 check whether source data is
empty
Posted by sh...@apache.org.
KYLIN-2004 check whether source data is empty
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56136ede
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56136ede
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56136ede
Branch: refs/heads/v1.5.4-release2
Commit: 56136ede7c8b9abac5ddd7b7785b3f63c59b74db
Parents: 233a699
Author: shaofengshi <sh...@apache.org>
Authored: Sat Sep 10 17:52:32 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Sep 10 17:59:59 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/source/hive/HiveMRInput.java | 37 ++++++++++----------
1 file changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56136ede/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 3ea9af5..520d7cc 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -281,23 +281,6 @@ public class HiveMRInput implements IMRInput {
}
}
- private int determineNumReducer(KylinConfig config) throws IOException {
- computeRowCount(config.getCliCommandExecutor());
-
- Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
- long rowCount = readRowCountFromFile(rowCountFile);
- int mapperInputRows = config.getHadoopJobMapperInputRows();
-
- int numReducers = Math.round(rowCount / ((float) mapperInputRows));
- numReducers = Math.max(1, numReducers);
-
- stepLogger.log("total input rows = " + rowCount);
- stepLogger.log("expected input rows per mapper = " + mapperInputRows);
- stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
-
- return numReducers;
- }
-
private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(getInitStatement());
@@ -327,7 +310,25 @@ public class HiveMRInput implements IMRInput {
KylinConfig config = getCubeSpecificConfig();
try {
- int numReducers = determineNumReducer(config);
+
+ computeRowCount(config.getCliCommandExecutor());
+
+ Path rowCountFile = new Path(getRowCountOutputDir(), "000000_0");
+ long rowCount = readRowCountFromFile(rowCountFile);
+ if (!config.isEmptySegmentAllowed() && rowCount == 0) {
+ stepLogger.log("Detect upstream hive table is empty, " + "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
+ return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog());
+ }
+
+ int mapperInputRows = config.getHadoopJobMapperInputRows();
+
+ int numReducers = Math.round(rowCount / ((float) mapperInputRows));
+ numReducers = Math.max(1, numReducers);
+
+ stepLogger.log("total input rows = " + rowCount);
+ stepLogger.log("expected input rows per mapper = " + mapperInputRows);
+ stepLogger.log("num reducers for RedistributeFlatHiveTableStep = " + numReducers);
+
redistributeTable(config, numReducers);
return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());
[24/43] kylin git commit: KYLIN 1997 add pivot feature back in query
result page
Posted by sh...@apache.org.
KYLIN 1997 add pivot feature back in query result page
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be7751bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be7751bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be7751bc
Branch: refs/heads/v1.5.4-release2
Commit: be7751bc021063f89e5955e5f1c50d66bedc5b91
Parents: ded3b58
Author: chenzhx <34...@qq.com>
Authored: Thu Sep 8 16:21:32 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Thu Sep 8 17:51:28 2016 +0800
----------------------------------------------------------------------
webapp/app/js/app.js | 2 +-
webapp/app/partials/query/query_detail.html | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/be7751bc/webapp/app/js/app.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/app.js b/webapp/app/js/app.js
index 3708037..629617e 100644
--- a/webapp/app/js/app.js
+++ b/webapp/app/js/app.js
@@ -17,4 +17,4 @@
*/
//Kylin Application Module
-KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.grid', 'ui.grid.resizeColumns', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable', 'angularBootstrapNavTree', 'toggle-switch', 'ngSanitize', 'ui.select', 'ui.bootstrap.datetimepicker']);
+KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.grid', 'ui.grid.resizeColumns', 'ui.grid.grouping', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable', 'angularBootstrapNavTree', 'toggle-switch', 'ngSanitize', 'ui.select', 'ui.bootstrap.datetimepicker']);
http://git-wip-us.apache.org/repos/asf/kylin/blob/be7751bc/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index 72da6b7..8e1286c 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -96,7 +96,7 @@
<div ng-if="!curQuery.graph.show">
<div class="query-results">
<div ng-if="curQuery.status=='success'">
- <div class="gridStyle" ui-grid="curQuery.result.gridOptions" ui-grid-resize-columns
+ <div class="gridStyle" ui-grid="curQuery.result.gridOptions" ui-grid-resize-columns ui-grid-grouping
ng-if="curQuery.result.results.length > 0" id="data_grid"
style="{{ui.fullScreen?'height: 600px;width:auto':'height: 300px'}}"></div>
<div ng-if="!curQuery.result.results || curQuery.result.results.length == 0"