You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/12/30 10:49:47 UTC
kylin git commit: Update sample kylin_sales cube to v3
Repository: kylin
Updated Branches:
refs/heads/2.x-staging 8d76b7604 -> a95cef67d
Update sample kylin_sales cube to v3
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a95cef67
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a95cef67
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a95cef67
Branch: refs/heads/2.x-staging
Commit: a95cef67d02a91314e62155ca8eed23aa4ce806b
Parents: 8d76b76
Author: shaofengshi <sh...@apache.org>
Authored: Wed Dec 30 17:49:07 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Dec 30 17:49:38 2015 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctHiveColumnsMapper.java | 4 -
.../metadata/cube/kylin_sales_cube.json | 2 +-
.../cube_desc/kylin_sales_cube_desc.json | 362 ++++++++-----------
3 files changed, 153 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a95cef67/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index 8d6191a..d8880dd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -110,19 +110,15 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
String[] row = flatTableInputFormat.parseMapperInput(record);
try {
for (int i = 0; i < factDictCols.size(); i++) {
-// outputKey.set(Bytes.toBytes((long)factDictCols.get(i)));
String fieldValue = row[dictionaryColumnIndex[i]];
if (fieldValue == null)
continue;
-// outputValue.set(bytes, 0, bytes.length);
keyBuffer.clear();
keyBuffer.putLong((long)factDictCols.get(i));
keyBuffer.put(Bytes.toBytes(fieldValue));
outputKey.set(keyBuffer.array(), 0, keyBuffer.position());
context.write(outputKey, EMPTY_TEXT);
-
-// context.write(outputKey, outputValue);
}
} catch (Exception ex) {
handleErrorRecord(row, ex);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a95cef67/examples/sample_cube/metadata/cube/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube/kylin_sales_cube.json b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
index affd387..fb8affa 100644
--- a/examples/sample_cube/metadata/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
@@ -1,5 +1,5 @@
{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
+ "uuid" : "2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b",
"last_modified" : 0,
"name" : "kylin_sales_cube",
"owner" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/a95cef67/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
index 9320aaf..93195a7 100644
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
@@ -1,225 +1,167 @@
{
- "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "kylin_sales_cube_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "DEFAULT.KYLIN_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 2,
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.KYLIN_SALES",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
+ "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "kylin_sales_cube_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "DEFAULT.KYLIN_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "decimal(19,4)"
},
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "bigint"
},
- {
- "id": 5,
- "name": "SELLER_CNT_HLL",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID"
- },
- "returntype": "hllc(10)"
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_CNT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
},
- "dependent_measure_ref": null
+ "returntype" : "hllc(10)"
},
- {
- "id": 6,
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME"
- },
- "returntype": "hllc(10)"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "part_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_FORMAT_CNT",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
},
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "part_dt",
- "lstg_site_id",
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl3_name",
- "categ_lvl2_name",
- "lstg_format_name"
- ]
- ]
+ "returntype" : "hllc(10)"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "part_dt",
+ "encoding" : "dict"
+ }, {
+ "column" : "leaf_categ_id",
+ "encoding" : "dict"
+ }, {
+ "column" : "meta_categ_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl2_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "categ_lvl3_name",
+ "encoding" : "dict"
+ }, {
+ "column" : "lstg_format_name",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "lstg_site_id",
+ "encoding" : "dict"
+ } ]
},
- "last_modified": 1426255280419,
- "model_name": "kylin_sales_model",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "seller_cnt_hll",
- "seller_format_cnt"
- ]
- }
- ]
- }
- ]
+ "signature" : "GpFEh87Y465FiHgu1UFmhw==",
+ "last_modified" : 1451468470824,
+ "model_name" : "kylin_sales_model",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "seller_cnt_hll", "seller_format_cnt" ]
+ } ]
+ } ]
},
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
+ "aggregation_groups" : [ {
+ "includes" : [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", "part_dt" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : { }
}
\ No newline at end of file