You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/15 05:06:13 UTC
[01/50] [abbrv] incubator-kylin git commit: KYLIN-748 bug fix
Repository: incubator-kylin
Updated Branches:
refs/heads/streaming-localdict c7343c01b -> 4cf65f070
KYLIN-748 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/142fb37c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/142fb37c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/142fb37c
Branch: refs/heads/streaming-localdict
Commit: 142fb37c9148d5a2d30135524ebd061dc19ff575
Parents: bab5944
Author: honma <ho...@ebay.com>
Authored: Mon May 4 17:37:12 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 4 17:37:21 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/142fb37c/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 1a3a1c0..b70ef86 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -135,7 +135,7 @@ public class ColumnDesc {
if (id != null)
zeroBasedIndex = Integer.parseInt(id) - 1;
- type = DataType.getInstance(datatype);
+ this.setDatatype(DataType.getInstance(datatype).toString());
}
public boolean isSameAs(String tableName, String columnName) {
[16/50] [abbrv] incubator-kylin git commit: KYLIN-749 add
STEAMING_TABLE def
Posted by sh...@apache.org.
KYLIN-749 add STEAMING_TABLE def
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/23c61d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/23c61d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/23c61d26
Branch: refs/heads/streaming-localdict
Commit: 23c61d262a720644b280daf0952f4fd38aabc3c8
Parents: eeb2abe
Author: honma <ho...@ebay.com>
Authored: Mon May 4 15:21:21 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:16:48 2015 +0800
----------------------------------------------------------------------
.../invertedindex/test_streaming_table.json | 33 ++++++++
.../test_streaming_table_desc.json | 25 ++++++
.../test_kylin_left_join_model_desc.json | 87 +++++++++++---------
.../test_streaming_table_model_desc.json | 29 +++++++
.../localmeta/project/default.json | 5 ++
.../localmeta/table/DEFAULT.SREAMING_TABLE.json | 43 ++++++++++
6 files changed, 184 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
new file mode 100644
index 0000000..dc341aa
--- /dev/null
+++ b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
@@ -0,0 +1,33 @@
+{
+ "uuid": "daa53e80-41be-49a5-89ca-9fb729999812",
+ "name": "test_streaming_table",
+ "owner": null,
+ "version": null,
+ "cost": 10,
+ "status": "READY",
+ "segments": [
+ {
+ "uuid": null,
+ "name": "19700101000000_20140901000000",
+ "status": "READY",
+ "dictionaries": {
+ },
+ "storage_location_identifier": "",
+ "date_range_start": 0,
+ "date_range_end": 0,
+ "size_kb": 0,
+ "input_records": 0,
+ "input_records_size": 0,
+ "last_build_time": 0,
+ "last_build_job_id": null,
+ "create_time": null,
+ "binary_signature": null
+ }
+ ],
+ "last_modified": 0,
+ "descriptor": "test_streaming_table_desc",
+ "create_time": null,
+ "input_records_size": 0,
+ "size_kb": 0,
+ "source_records_count": 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
new file mode 100644
index 0000000..27cb053
--- /dev/null
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
@@ -0,0 +1,25 @@
+{
+ "uuid": "74bf87b5-c7b5-4420-a12a-07f6b3864789",
+ "last_modified": 0,
+ "name": "test_streaming_table_desc",
+ "model_name": "test_streaming_table_model_desc",
+ "timestamp_dimension": "ts",
+ "value_dimensions": [
+ {
+ "table": "default.streaming_table",
+ "columns": [
+ "ts",
+ "minute_start",
+ "hour_start",
+ "itm",
+ "site"
+ ]
+ }
+ ],
+ "metrics": [
+ "gmv",
+ "item_count"
+ ],
+ "sharding": 1,
+ "slice_size": 1000
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index fa11c33..d05a08f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -1,35 +1,46 @@
{
- "uuid" : "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
- "name" : "test_kylin_left_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "left",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "left",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
+ "uuid": "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
+ "name": "test_kylin_left_join_model_desc",
+ "lookups": [
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "left",
+ "primary_key": ["CAL_DT"],
+ "foreign_key": ["CAL_DT"]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "LEAF_CATEG_ID",
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LEAF_CATEG_ID",
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "left",
+ "primary_key": ["SITE_ID"],
+ "foreign_key": ["LSTG_SITE_ID"]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "left",
+ "primary_key": ["SELLER_TYPE_CD"],
+ "foreign_key": ["SLR_SEGMENT_CD"]
+ }
}
- } ],
+ ],
"dimensions": [
{
"table": "default.test_kylin_fact",
@@ -89,13 +100,13 @@
"PRICE",
"ITEM_COUNT"
],
- "capacity" : "MEDIUM",
- "last_modified" : 1422435345352,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
+ "capacity": "MEDIUM",
+ "last_modified": 1422435345352,
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
new file mode 100644
index 0000000..66245ae
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
@@ -0,0 +1,29 @@
+{
+ "uuid": "ff527b94-f860-44c3-8452-93b177888732",
+ "name": "test_streaming_table_model_desc",
+ "dimensions": [
+ {
+ "table": "default.streaming_table",
+ "columns": [
+ "ts",
+ "minute_start",
+ "hour_start",
+ "itm",
+ "site"
+ ]
+ }
+ ],
+ "metrics": [
+ "gmv",
+ "item_count"
+ ],
+ "capacity": "MEDIUM",
+ "last_modified": 0,
+ "fact_table": "default.streaming_table",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "default.streaming_table.ts",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 55d8309..a11d4ca 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -33,6 +33,11 @@
"realization": "test_kylin_ii_inner_join"
},
{
+ "name": "test_streaming_table",
+ "type": "INVERTED_INDEX",
+ "realization": "test_streaming_table"
+ },
+ {
"name": "test_kylin_hybrid_left_join",
"type": "HYBRID",
"realization": "test_kylin_hybrid_left_join"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/23c61d26/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
new file mode 100644
index 0000000..0e001c9
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
@@ -0,0 +1,43 @@
+{
+ "uuid": "e286e39e-40d7-44c2-8fa2-41b365123987",
+ "name": "STREAMING_TABLE",
+ "columns": [
+ {
+ "id": "1",
+ "name": "ts",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "2",
+ "name": "minute_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "3",
+ "name": "hour_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "4",
+ "name": "itm",
+ "datatype": "string"
+ },
+ {
+ "id": "5",
+ "name": "site",
+ "datatype": "string"
+ },
+ {
+ "id": "6",
+ "name": "gmv",
+ "datatype": "decimal(19,6)"
+ },
+ {
+ "id": "7",
+ "name": "item_count",
+ "datatype": "bigint"
+ }
+ ],
+ "database": "DEFAULT",
+ "last_modified": 0
+}
\ No newline at end of file
[05/50] [abbrv] incubator-kylin git commit: KYLIN-748
restoreDecimalPoint revisited
Posted by sh...@apache.org.
KYLIN-748 restoreDecimalPoint revisited
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6533a33c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6533a33c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6533a33c
Branch: refs/heads/streaming-localdict
Commit: 6533a33c81342457d41ff81ebd46a03f1dc1c358
Parents: 0ab0166
Author: honma <ho...@ebay.com>
Authored: Tue May 5 13:35:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 13:35:30 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/measure/fixedlen/FixedPointLongCodec.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6533a33c/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
index a0409b5..5218101 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
@@ -70,15 +70,12 @@ public class FixedPointLongCodec extends FixedLenMeasureCodec<LongWritable> {
}
String restoreDecimalPoint(long value) {
- String valueStr = Long.toString(value);
- if (scale < 0 || scale >= valueStr.length()) {
+ if (scale < 0) {
throw new RuntimeException("Bad scale: " + scale + " with value: " + value);
} else if (scale == 0) {
return Long.toString(value);
} else {
- StringBuilder sb = new StringBuilder(valueStr);
- sb.insert(valueStr.length() - scale, '.');
- return sb.toString();
+ return String.format("%." + scale + "f", value / (Math.pow(10, scale)));
}
}
[21/50] [abbrv] incubator-kylin git commit: KYLIN-754 - Enable lzo on
II's table
Posted by sh...@apache.org.
KYLIN-754 - Enable lzo on II's table
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e349cda6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e349cda6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e349cda6
Branch: refs/heads/streaming-localdict
Commit: e349cda6693eb3e897df4abb930e82babd479c63
Parents: 9854f7d
Author: honma <ho...@ebay.com>
Authored: Thu May 7 15:51:45 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:51:45 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e349cda6/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 77d0d55..cf8d005 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
@@ -65,7 +66,7 @@ public class IICreateHTableJob extends AbstractHadoopJob {
HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor cf = new HColumnDescriptor(IIDesc.HBASE_FAMILY);
cf.setMaxVersions(1);
- //cf.setCompressionType(Algorithm.LZO);
+ cf.setCompressionType(Compression.Algorithm.LZO);
cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
tableDesc.addFamily(cf);
tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());
[02/50] [abbrv] incubator-kylin git commit: KYLIN-748 refine
SqlHiveDataTypeMapping
Posted by sh...@apache.org.
KYLIN-748 refine SqlHiveDataTypeMapping
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b376b60c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b376b60c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b376b60c
Branch: refs/heads/streaming-localdict
Commit: b376b60cb831716c4760e8a69651a8d1c6a845b4
Parents: 142fb37
Author: honma <ho...@ebay.com>
Authored: Mon May 4 18:12:28 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 4 18:12:28 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/hive/SqlHiveDataTypeMapping.java | 17 +----------------
1 file changed, 1 insertion(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b376b60c/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
index f283501..0aec98f 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
@@ -18,29 +18,14 @@
package org.apache.kylin.job.hadoop.hive;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* @author George Song (ysong1)
*
*/
public class SqlHiveDataTypeMapping {
- private static final Map<String, String> sqlToHiveDataTypeMapping = new HashMap<String, String>();
-
- static {
- sqlToHiveDataTypeMapping.put("short", "smallint");
- sqlToHiveDataTypeMapping.put("long", "bigint");
- sqlToHiveDataTypeMapping.put("byte", "tinyint");
- sqlToHiveDataTypeMapping.put("datetime", "date");
- }
-
public static String getHiveDataType(String javaDataType) {
- String hiveDataType = sqlToHiveDataTypeMapping.get(javaDataType.toLowerCase());
- if (hiveDataType == null) {
- hiveDataType = javaDataType;
- }
+ String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
return hiveDataType.toLowerCase();
}
}
[34/50] [abbrv] incubator-kylin git commit: KYLIN-702 Move hive
related properties to kylin_hive_conf.xml
Posted by sh...@apache.org.
KYLIN-702 Move hive related properties to kylin_hive_conf.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/33f8fee6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/33f8fee6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/33f8fee6
Branch: refs/heads/streaming-localdict
Commit: 33f8fee6e081f25b010eb47bf74f968696c66581
Parents: d1e4b9b
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue May 12 14:10:18 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue May 12 14:10:35 2015 +0800
----------------------------------------------------------------------
.../test_case_data/sandbox/kylin_hive_conf.xml | 36 ++++++++++++++++++++
.../test_case_data/sandbox/kylin_job_conf.xml | 18 ----------
2 files changed, 36 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/33f8fee6/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index 053525e..d990175 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -1,6 +1,42 @@
<?xml version="1.0"?>
<configuration>
<property>
+ <name>dfs.replication</name>
+ <value>2</value>
+ <description>Block replication</description>
+ </property>
+
+ <property>
+ <name>hive.exec.compress.output</name>
+ <value>true</value>
+ <description>enable compress</description>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask</name>
+ <value>true</value>
+ <description>enable map-side join</description>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>300000000</value>
+ <description>enable map-side join</description>
+ </property>
+
+ <!--
+ <property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description></description>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description></description>
+ </property>
+ -->
+ <property>
<name>hive.merge.mapfiles</name>
<value>true</value>
<description>Enable hive file merge on mapper only job</description>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/33f8fee6/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index 92cf745..f811ed4 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -58,22 +58,4 @@
<description>Block replication</description>
</property>
-
- <property>
- <name>hive.merge.mapfiles</name>
- <value>true</value>
- <description>Enable hive file merge on mapper only job</description>
- </property>
-
- <property>
- <name>hive.merge.mapredfiles</name>
- <value>true</value>
- <description>Enable hive file merge on map-reduce job</description>
- </property>
-
- <property>
- <name>hive.merge.size.per.task</name>
- <value>16000000</value>
- <description>Size for the merged file: 16M</description>
- </property>
</configuration>
\ No newline at end of file
[44/50] [abbrv] incubator-kylin git commit: KYLIN-766
Posted by sh...@apache.org.
KYLIN-766
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/69240574
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/69240574
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/69240574
Branch: refs/heads/streaming-localdict
Commit: 69240574c874a830ea406edeab881a38b3e32040
Parents: fc056fd
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu May 14 16:38:09 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu May 14 16:38:09 2015 +0800
----------------------------------------------------------------------
.../serializer/BigDecimalSerializer.java | 20 ++++++--
.../serializer/BigDecimalSerializerTest.java | 52 ++++++++++++++++++++
2 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69240574/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
index 671d249..cf146fb 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
@@ -18,19 +18,23 @@
package org.apache.kylin.metadata.serializer;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
/**
* @author yangli9
*
*/
public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
+
+ private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
final DataType type;
final int maxLength;
@@ -43,9 +47,14 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
@Override
public void serialize(BigDecimal value, ByteBuffer out) {
+ if (value.scale() > type.getScale()) {
+ logger.warn("value's scale has exceeded the " + type.getScale() +", cut it off, to ensure encoded value do not exceed maxLength " + maxLength);
+ value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
+ }
byte[] bytes = value.unscaledValue().toByteArray();
- if (bytes.length + 2 > maxLength)
+ if (bytes.length + 2 > maxLength) {
throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
BytesUtil.writeVInt(value.scale(), out);
BytesUtil.writeVInt(bytes.length, out);
@@ -89,4 +98,5 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
else
return new BigDecimal(Bytes.toString(value));
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69240574/metadata/src/test/java/org/apache/kylin/metadata/serializer/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/serializer/BigDecimalSerializerTest.java b/metadata/src/test/java/org/apache/kylin/metadata/serializer/BigDecimalSerializerTest.java
new file mode 100644
index 0000000..e7b7916
--- /dev/null
+++ b/metadata/src/test/java/org/apache/kylin/metadata/serializer/BigDecimalSerializerTest.java
@@ -0,0 +1,52 @@
+package org.apache.kylin.metadata.serializer;
+
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ */
+public class BigDecimalSerializerTest {
+
+ private static BigDecimalSerializer bigDecimalSerializer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ bigDecimalSerializer = new BigDecimalSerializer(DataType.getInstance("decimal"));
+ }
+
+ @Test
+ public void testNormal() {
+ BigDecimal input = new BigDecimal("1234.1234");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testScaleOutOfRange() {
+ BigDecimal input = new BigDecimal("1234.1234567890");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutOfPrecision() {
+ BigDecimal input = new BigDecimal("66855344214907231736.4924");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ bigDecimalSerializer.serialize(input, buffer);
+ }
+
+}
[23/50] [abbrv] incubator-kylin git commit: KYLIN-756 enforce all
customized jvm settings located in setenv.sh
Posted by sh...@apache.org.
KYLIN-756 enforce all customized jvm settings located in setenv.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/32ceeee7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/32ceeee7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/32ceeee7
Branch: refs/heads/streaming-localdict
Commit: 32ceeee715318c36bee47639c5c4ac864eb687db
Parents: 50c31c6
Author: honma <ho...@ebay.com>
Authored: Thu May 7 16:06:26 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 16:06:26 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 8 ++++----
bin/setenv.sh | 6 ++++--
2 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/32ceeee7/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 532900b..9386ce8 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -37,9 +37,9 @@ then
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
- export JAVA_OPTS="-Xms2048M -Xmx2048M -XX:MaxPermSize=512m"
- hbase ${JAVA_OPTS} ${KYLIN_EXTRA_START_OPTS} \
+ # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
+ hbase ${KYLIN_EXTRA_START_OPTS} \
-Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
@@ -101,9 +101,9 @@ then
mkdir -p ${KYLIN_HOME}/ext
export HBASE_CLASSPATH=$hive_dependency:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}
- export JAVA_OPTS="-Xms4g -Xmx4g -XX:MaxPermSize=512m"
- hbase ${JAVA_OPTS} ${KYLIN_EXTRA_START_OPTS} \
+ # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
+ hbase ${KYLIN_EXTRA_START_OPTS} \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
-Dkylin.hive.dependency=${hive_dependency} \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/32ceeee7/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/setenv.sh b/bin/setenv.sh
index 53f614d..1d485c2 100755
--- a/bin/setenv.sh
+++ b/bin/setenv.sh
@@ -1,12 +1,14 @@
#!/bin/bash
+# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
# uncomment following to for it to take effect
+export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -XX:MaxPermSize=128M"
# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
-# uncomment following to for it to take effect
+# uncomment following to for it to take effect(the values need adjusting to fit your env)
# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:${KYLIN_HOME}/lib/CustomAgent.jar -Dcom.ebay.lnp.rmiagent.port=8080"
-# uncomment following to for it to take effect
+# uncomment following to for it to take effect(the values need adjusting to fit your env)
# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
export KYLIN_EXTRA_START_OPTS=""
[43/50] [abbrv] incubator-kylin git commit: KYLIN-763 remove author
information
Posted by sh...@apache.org.
KYLIN-763 remove author information
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fc056fda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fc056fda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fc056fda
Branch: refs/heads/streaming-localdict
Commit: fc056fda2a276daebad9795ab3d557f225803b0c
Parents: 12bbce6
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed May 13 16:36:37 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed May 13 16:36:46 2015 +0800
----------------------------------------------------------------------
common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java | 1 -
common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java | 1 -
.../kylin/common/restclient/CaseInsensitiveStringCache.java | 1 -
.../org/apache/kylin/common/util/CaseInsensitiveStringMap.java | 1 -
.../main/java/org/apache/kylin/common/util/CompressionUtils.java | 1 -
.../src/main/java/org/apache/kylin/common/util/FIFOIterable.java | 1 -
.../src/main/java/org/apache/kylin/common/util/FIFOIterator.java | 1 -
.../main/java/org/apache/kylin/common/util/LogTitlePrinter.java | 1 -
.../src/main/java/org/apache/kylin/common/util/PartialSorter.java | 1 -
common/src/main/java/org/apache/kylin/common/util/RangeUtil.java | 1 -
common/src/main/java/org/apache/kylin/common/util/SortUtil.java | 1 -
common/src/main/java/org/apache/kylin/common/util/SoutLogger.java | 1 -
common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java | 1 -
common/src/main/java/org/apache/kylin/common/util/TimeUtil.java | 1 -
common/src/test/java/org/apache/kylin/common/util/BasicTest.java | 1 -
.../src/test/java/org/apache/kylin/common/util/HbaseImporter.java | 1 -
.../org/apache/kylin/common/util/InstallJarIntoMavenTest.java | 1 -
.../test/java/org/apache/kylin/common/util/PartialSorterTest.java | 1 -
common/src/test/java/org/apache/kylin/common/util/RangeTest.java | 1 -
.../src/test/java/org/apache/kylin/common/util/RangeUtilTest.java | 1 -
.../src/test/java/org/apache/kylin/common/util/TimeUtilTest.java | 1 -
.../main/java/org/apache/kylin/cube/CubeCapabilityChecker.java | 1 -
.../src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java | 1 -
cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java | 1 -
.../src/main/java/org/apache/kylin/dict/DictionarySerializer.java | 1 -
.../src/main/java/org/apache/kylin/dict/IDictionaryAware.java | 3 +--
.../src/main/java/org/apache/kylin/dict/TimeStrDictionary.java | 1 -
.../test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java | 1 -
.../test/java/org/apache/kylin/dict/lookup/LookupTableTest.java | 1 -
.../java/org/apache/kylin/invertedindex/IICapabilityChecker.java | 1 -
.../org/apache/kylin/invertedindex/index/BatchSliceMaker.java | 1 -
.../java/org/apache/kylin/invertedindex/index/RawTableRecord.java | 1 -
.../apache/kylin/invertedindex/index/TableRecordInfoDigest.java | 1 -
.../apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java | 1 -
.../kylin/invertedindex/model/IIKeyValueCodecWithState.java | 1 -
.../src/main/java/org/apache/kylin/invertedindex/model/IIRow.java | 1 -
.../java/org/apache/kylin/invertedindex/model/KeyValueCodec.java | 1 -
.../org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java | 1 -
.../kylin/invertedindex/invertedindex/IIDescManagerTest.java | 1 -
.../org/apache/kylin/invertedindex/invertedindex/IIDescTest.java | 1 -
.../apache/kylin/invertedindex/invertedindex/IIInstanceTest.java | 1 -
.../org/apache/kylin/invertedindex/invertedindex/LZFTest.java | 1 -
job/src/main/java/org/apache/kylin/job/Scheduler.java | 1 -
job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java | 1 -
.../java/org/apache/kylin/job/common/HadoopShellExecutable.java | 1 -
job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java | 1 -
.../java/org/apache/kylin/job/common/MapReduceExecutable.java | 1 -
.../main/java/org/apache/kylin/job/common/ShellExecutable.java | 1 -
.../java/org/apache/kylin/job/constant/ExecutableConstants.java | 1 -
job/src/main/java/org/apache/kylin/job/cube/CubingJob.java | 1 -
job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java | 1 -
.../org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java | 1 -
.../org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java | 1 -
job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 1 -
.../main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java | 1 -
job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java | 1 -
.../org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java | 1 -
.../java/org/apache/kylin/job/exception/ExecuteException.java | 1 -
.../apache/kylin/job/exception/IllegalStateTranferException.java | 1 -
.../main/java/org/apache/kylin/job/exception/LockException.java | 1 -
.../java/org/apache/kylin/job/exception/PersistentException.java | 1 -
.../java/org/apache/kylin/job/exception/SchedulerException.java | 1 -
.../java/org/apache/kylin/job/execution/AbstractExecutable.java | 1 -
.../java/org/apache/kylin/job/execution/ChainedExecutable.java | 1 -
.../org/apache/kylin/job/execution/DefaultChainedExecutable.java | 1 -
.../main/java/org/apache/kylin/job/execution/DefaultOutput.java | 1 -
job/src/main/java/org/apache/kylin/job/execution/Executable.java | 1 -
.../java/org/apache/kylin/job/execution/ExecutableContext.java | 1 -
.../main/java/org/apache/kylin/job/execution/ExecutableState.java | 1 -
.../main/java/org/apache/kylin/job/execution/ExecuteResult.java | 1 -
job/src/main/java/org/apache/kylin/job/execution/Idempotent.java | 1 -
job/src/main/java/org/apache/kylin/job/execution/Output.java | 1 -
.../org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java | 1 -
.../kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java | 1 -
.../java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java | 1 -
.../java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java | 1 -
.../org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java | 1 -
.../org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java | 1 -
.../apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java | 1 -
.../apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java | 1 -
.../java/org/apache/kylin/job/impl/threadpool/DefaultContext.java | 1 -
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 1 -
job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java | 1 -
.../java/org/apache/kylin/job/invertedindex/IIJobBuilder.java | 1 -
.../main/java/org/apache/kylin/job/manager/ExecutableManager.java | 1 -
.../main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java | 1 -
.../java/org/apache/kylin/job/streaming/StreamingBootstrap.java | 1 -
.../main/java/org/apache/kylin/job/streaming/StreamingCLI.java | 1 -
job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java | 1 -
.../main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java | 1 -
.../main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java | 1 -
.../main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java | 1 -
.../java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java | 1 -
.../java/org/apache/kylin/job/tools/LZOSupportnessChecker.java | 1 -
job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java | 1 -
.../test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java | 1 -
job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java | 1 -
job/src/test/java/org/apache/kylin/job/DataGenTest.java | 1 -
job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java | 1 -
job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java | 1 -
job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java | 1 -
job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java | 1 -
job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java | 1 -
job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java | 1 -
.../java/org/apache/kylin/job/dataGen/FactTableGenerator.java | 1 -
job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java | 1 -
.../java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java | 1 -
.../java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java | 1 -
.../java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java | 1 -
.../test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java | 1 -
.../java/org/apache/kylin/job/hadoop/invertedindex/IITest.java | 1 -
.../org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java | 1 -
.../apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java | 1 -
.../java/org/apache/kylin/job/manager/ExecutableManagerTest.java | 1 -
.../test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java | 1 -
.../org/apache/kylin/metadata/filter/DateConditionModifier.java | 1 -
.../java/org/apache/kylin/metadata/filter/TsConditionEraser.java | 1 -
.../org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java | 1 -
.../main/java/org/apache/kylin/metadata/model/DimensionDesc.java | 1 -
.../org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java | 1 -
.../org/apache/kylin/metadata/model/IntermediateColumnDesc.java | 1 -
.../java/org/apache/kylin/metadata/project/RealizationEntry.java | 1 -
.../apache/kylin/metadata/realization/IRealizationConstants.java | 1 -
.../apache/kylin/metadata/realization/RealizationRegistry.java | 1 -
.../org/apache/kylin/metadata/realization/RealizationType.java | 1 -
.../java/org/apache/kylin/metadata/realization/SQLDigest.java | 1 -
.../java/org/apache/kylin/metadata/realization/SQLDigestUtil.java | 1 -
.../org/apache/kylin/metadata/realization/StreamSQLDigest.java | 1 -
.../org/apache/kylin/metadata/tuple/CompoundTupleIterator.java | 1 -
.../java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java | 1 -
.../java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java | 1 -
.../kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java | 1 -
.../test/java/org/apache/kylin/metadata/model/TableDescTest.java | 1 -
.../src/main/java/org/apache/kylin/query/routing/RoutingRule.java | 1 -
.../routing/RoutingRules/AdjustForWeaklyMatchedRealization.java | 1 -
.../kylin/query/routing/RoutingRules/RealizationPriorityRule.java | 1 -
.../kylin/query/routing/RoutingRules/RealizationSortRule.java | 1 -
.../routing/RoutingRules/RemoveUncapableRealizationsRule.java | 1 -
.../test/java/org/apache/kylin/query/test/CombinationTest.java | 1 -
query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java | 1 -
.../src/test/java/org/apache/kylin/query/test/KylinTestBase.java | 1 -
.../java/org/apache/kylin/query/test/RealizationRegistryTest.java | 1 -
server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java | 1 -
.../main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java | 1 -
server/src/main/java/org/apache/kylin/rest/model/TableMeta.java | 1 -
.../java/org/apache/kylin/rest/request/CreateProjectRequest.java | 1 -
.../src/main/java/org/apache/kylin/rest/request/MetaRequest.java | 1 -
.../java/org/apache/kylin/rest/request/UpdateProjectRequest.java | 1 -
.../src/main/java/org/apache/kylin/rest/service/CacheService.java | 1 -
server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java | 1 -
.../apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java | 1 -
.../org/apache/kylin/rest/controller/ProjectControllerTest.java | 1 -
.../test/java/org/apache/kylin/rest/service/CacheServiceTest.java | 1 -
.../org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java | 1 -
.../java/org/apache/kylin/storage/cache/StorageLayerCache.java | 1 -
.../main/java/org/apache/kylin/storage/cache/StreamSQLResult.java | 1 -
.../main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java | 1 -
.../java/org/apache/kylin/storage/gridtable/GTComboStore.java | 1 -
.../src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java | 1 -
.../org/apache/kylin/storage/gridtable/diskstore/FileSystem.java | 1 -
.../org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java | 1 -
.../kylin/storage/gridtable/diskstore/HadoopFileSystem.java | 1 -
.../apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java | 1 -
.../java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java | 1 -
.../apache/kylin/storage/hbase/coprocessor/AggregationCache.java | 1 -
.../kylin/storage/hbase/coprocessor/CoprocessorConstants.java | 1 -
.../apache/kylin/storage/hbase/coprocessor/FilterDecorator.java | 1 -
.../storage/hbase/coprocessor/endpoint/ClearTextDictionary.java | 1 -
.../hbase/coprocessor/endpoint/EndpointAggregationCache.java | 1 -
.../storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java | 1 -
.../storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java | 1 -
.../kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java | 1 -
.../storage/hbase/coprocessor/endpoint/IIResponseAdapter.java | 1 -
.../kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java | 1 -
.../storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java | 1 -
.../storage/hbase/coprocessor/endpoint/TsConditionExtractor.java | 1 -
.../main/java/org/apache/kylin/storage/hybrid/HybridInstance.java | 1 -
.../main/java/org/apache/kylin/storage/hybrid/HybridManager.java | 1 -
.../java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java | 1 -
.../src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java | 1 -
.../src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java | 1 -
.../apache/kylin/storage/filter/DateConditionModifierTest.java | 1 -
.../org/apache/kylin/storage/filter/TsConditionEraserTest.java | 1 -
.../hbase/coprocessor/endpoint/EndpointAggregationTest.java | 1 -
.../storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java | 1 -
.../hbase/coprocessor/endpoint/TsConditionExtractorTest.java | 1 -
.../java/org/apache/kylin/storage/hybrid/HybridManagerTest.java | 1 -
.../src/main/java/org/apache/kylin/streaming/BrokerConfig.java | 1 -
.../main/java/org/apache/kylin/streaming/JsonStreamParser.java | 1 -
.../src/main/java/org/apache/kylin/streaming/KafkaConfig.java | 1 -
.../src/main/java/org/apache/kylin/streaming/KafkaConsumer.java | 1 -
.../src/main/java/org/apache/kylin/streaming/KafkaRequester.java | 1 -
.../src/main/java/org/apache/kylin/streaming/StreamBuilder.java | 1 -
.../src/main/java/org/apache/kylin/streaming/StreamMessage.java | 1 -
.../src/main/java/org/apache/kylin/streaming/StreamParser.java | 1 -
.../main/java/org/apache/kylin/streaming/StreamingManager.java | 1 -
.../main/java/org/apache/kylin/streaming/StringStreamParser.java | 1 -
streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java | 1 -
.../org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java | 1 -
.../org/apache/kylin/streaming/invertedindex/SliceBuilder.java | 1 -
.../src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java | 1 -
.../test/java/org/apache/kylin/streaming/KafkaConsumerTest.java | 1 -
.../test/java/org/apache/kylin/streaming/KafkaRequesterTest.java | 1 -
.../java/org/apache/kylin/streaming/OneOffStreamProducer.java | 1 -
.../java/org/apache/kylin/streaming/StreamingManagerTest.java | 1 -
.../kylin/streaming/invertedindex/PrintOutStreamBuilder.java | 1 -
206 files changed, 1 insertion(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java b/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
index a070c56..a933782 100644
--- a/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
+++ b/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
/**
- * Created by Hongbin Ma(Binmahone) on 1/19/15.
*/
public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
protected void bindCurrentConfiguration(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java b/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
index aba411d..bc63df7 100644
--- a/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
+++ b/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
/**
- * Created by Hongbin Ma(Binmahone) on 1/19/15.
*/
public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
protected void bindCurrentConfiguration(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java b/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
index 14bc517..ceefc08 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
@@ -21,7 +21,6 @@ package org.apache.kylin.common.restclient;
import java.util.concurrent.ConcurrentSkipListMap;
/**
- * Created by qianzhou on 1/15/15.
*/
public class CaseInsensitiveStringCache<V> extends SingleValueCache<String, V>{
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java b/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
index ee64d51..d6de581 100644
--- a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
+++ b/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Set;
/**
- * Created by qianzhou on 12/3/14.
*/
public class CaseInsensitiveStringMap<T> implements Map<String, T> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java b/common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
index c6b0088..6dda7ec 100644
--- a/common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
+++ b/common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
@@ -9,7 +9,6 @@ import java.util.zip.Inflater;
import org.slf4j.LoggerFactory;
/**
- * Created by Hongbin Ma(Binmahone) on 4/24/15.
*/
public class CompressionUtils {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CompressionUtils.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/FIFOIterable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/FIFOIterable.java b/common/src/main/java/org/apache/kylin/common/util/FIFOIterable.java
index c0f7d68..4c4bc6b 100644
--- a/common/src/main/java/org/apache/kylin/common/util/FIFOIterable.java
+++ b/common/src/main/java/org/apache/kylin/common/util/FIFOIterable.java
@@ -4,7 +4,6 @@ import java.util.Iterator;
import java.util.Queue;
/**
- * Created by Hongbin Ma(Binmahone) on 3/27/15.
*/
public class FIFOIterable<T> implements Iterable<T> {
private Queue<T> q;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/FIFOIterator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/FIFOIterator.java b/common/src/main/java/org/apache/kylin/common/util/FIFOIterator.java
index 6751cb0..f734143 100644
--- a/common/src/main/java/org/apache/kylin/common/util/FIFOIterator.java
+++ b/common/src/main/java/org/apache/kylin/common/util/FIFOIterator.java
@@ -4,7 +4,6 @@ import java.util.Iterator;
import java.util.Queue;
/**
- * Created by Hongbin Ma(Binmahone) on 3/27/15.
*
* Normal iterators in Collections are fail-safe,
* i.e. adding elements to a queue will break current iterator.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java b/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
index ffbc7c8..f7d3a16 100644
--- a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
+++ b/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
@@ -19,7 +19,6 @@
package org.apache.kylin.common.util;
/**
- * Created by Hongbin Ma(Binmahone) on 1/27/15.
*/
public class LogTitlePrinter {
public static void printTitle(String title) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java b/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
index 6d0723c..f73f525 100644
--- a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
+++ b/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
@@ -25,7 +25,6 @@ import java.util.Comparator;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*
* This utility class sorts only the specified part of a list
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
index d781e17..33d6770 100644
--- a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -7,7 +7,6 @@ import java.util.NavigableMap;
import com.google.common.collect.*;
/**
- * Created by Hongbin Ma(Binmahone) on 4/14/15.
*/
public class RangeUtil {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
index ac9f216..f0d7cdb 100644
--- a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
@@ -6,7 +6,6 @@ import com.google.common.collect.TreeMultimap;
import java.util.Iterator;
/**
- * Created by Hongbin Ma(Binmahone) on 5/5/15.
*/
public class SortUtil {
public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
index 3bd9090..cfbfe5c 100644
--- a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
+++ b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
@@ -1,7 +1,6 @@
package org.apache.kylin.common.util;
/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
*/
public class SoutLogger implements Logger {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
index 2c19665..c493424 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
@@ -25,7 +25,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
- * Created by Hongbin Ma(Binmahone) on 12/31/14.
*/
public class ThreadUtil {
@SuppressWarnings("unused")
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index 01fc2c1..26e3e06 100644
--- a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -1,7 +1,6 @@
package org.apache.kylin.common.util;
/**
- * Created by Hongbin Ma(Binmahone) on 5/4/15.
*/
public class TimeUtil {
private static int ONE_MINUTE_TS = 60 * 1000;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index c259507..2f4943d 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -33,7 +33,6 @@ import java.util.ArrayList;
import java.util.Calendar;
/**
-* Created by honma on 10/17/14.
* <p/>
* Keep this test case to test basic java functionality
* development concept proving use
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
index e1bac24..242fa57 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
@@ -18,7 +18,6 @@ import org.apache.kylin.common.persistence.HBaseConnection;
import com.google.common.base.Preconditions;
/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
*/
public class HbaseImporter {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java b/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
index 3ee1dd8..6dcdaf5 100644
--- a/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
@@ -27,7 +27,6 @@ import java.util.regex.Pattern;
import org.junit.Ignore;
/**
- * Created by honma on 6/6/14.
*/
public class InstallJarIntoMavenTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java b/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
index 728186b..e24e7f8 100644
--- a/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
@@ -25,7 +25,6 @@ import java.util.Comparator;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class PartialSorterTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/RangeTest.java b/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
index 62d7b63..2a9ce40 100644
--- a/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
@@ -8,7 +8,6 @@ import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
/**
- * Created by Hongbin Ma(Binmahone) on 4/14/15.
*/
public class RangeTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
index 1bd8403..84688aa 100644
--- a/common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
@@ -9,7 +9,6 @@ import java.util.NavigableMap;
import java.util.TreeMap;
/**
- * Created by Hongbin Ma(Binmahone) on 4/20/15.
*/
public class RangeUtilTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index 1442412..90a0c40 100644
--- a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -8,7 +8,6 @@ import org.junit.Assert;
import org.junit.Test;
/**
- * Created by Hongbin Ma(Binmahone) on 5/4/15.
*/
public class TimeUtilTest {
public static long normalizeTime(long timeMillis, NormalizeUnit unit) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ab585c3..52a57b0 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -33,7 +33,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
*/
public class CubeCapabilityChecker {
private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
index dcdcfd5..f028c33 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.HashSet;
/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
*
* the unified logic for defining a sql's dimension
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index 0cb77df..973998f 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -35,7 +35,6 @@ import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by lukhan on 1/2/14.
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class RowKeyDesc {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
index 4b61591..a8ad2fa 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -6,7 +6,6 @@ import org.apache.kylin.common.util.ClassUtil;
import java.io.*;
/**
- * Created by qianzhou on 5/5/15.
*/
public final class DictionarySerializer {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java b/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
index 8f3473a..1c7a009 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
@@ -21,8 +21,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by Hongbin Ma(Binmahone) on 12/17/14.
- *
+ *
* Class that implement this interface has the ability to help dictionary encoding and decoding
*/
public interface IDictionaryAware {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index aef26ec..fde17a8 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -5,7 +5,6 @@ import org.apache.kylin.common.util.DateFormat;
import java.io.*;
/**
- * Created by Hongbin Ma(Binmahone) on 5/6/15.
*/
public class TimeStrDictionary extends Dictionary<String> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
index e723126..92f62fb 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
@@ -8,7 +8,6 @@ import org.junit.Test;
import java.util.Date;
/**
- * Created by Hongbin Ma(Binmahone) on 5/6/15.
*/
public class TimeStrDictionaryTests {
TimeStrDictionary dict;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 138ae33..bb4a717 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -34,7 +34,6 @@ import java.util.List;
import java.util.Set;
/**
- * Created by shaoshi on 2/3/15.
*/
public class LookupTableTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
index 5acba07..ca51492 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
@@ -31,7 +31,6 @@ import java.util.Collection;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
*/
public class IICapabilityChecker {
private static final Logger logger = LoggerFactory.getLogger(IICapabilityChecker.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java
index 2c3e593..0682d2b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java
@@ -41,7 +41,6 @@ import org.apache.kylin.invertedindex.model.IIDesc;
import java.util.List;
/**
- * Created by qianzhou on 3/20/15.
*/
public class BatchSliceMaker {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index 55104b5..b72ad26 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.io.LongWritable;
import java.util.Arrays;
/**
- * Created by honma on 11/10/14.
*/
public class RawTableRecord implements Cloneable {
TableRecordInfoDigest digest;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index d67c807..65b2a21 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -29,7 +29,6 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
/**
- * Created by honma on 11/10/14.
*/
public class TableRecordInfoDigest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
index 14934dc..b01ffa7 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
@@ -28,7 +28,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
*/
public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
index f3431df..c6ceea9 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
@@ -12,7 +12,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone) on 3/26/15.
*/
public class IIKeyValueCodecWithState extends IIKeyValueCodec {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
index 273d1e6..bf4880d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
@@ -43,7 +43,6 @@ import org.apache.kylin.common.util.BytesUtil;
import java.util.List;
/**
- * Created by qianzhou on 3/10/15.
*/
public final class IIRow {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
index bbd8d61..c151beb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
@@ -39,7 +39,6 @@ import org.apache.kylin.invertedindex.index.Slice;
import java.util.Collection;
/**
- * Created by qianzhou on 3/12/15.
*/
public interface KeyValueCodec {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
index 45e8058..24bd0de 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
@@ -47,7 +47,6 @@ import java.util.Collection;
import java.util.List;
/**
- * Created by qianzhou on 4/9/15.
*/
public final class IIDictionaryBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
index 7089862..a13a0d1 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
@@ -30,7 +30,6 @@ import org.junit.Before;
import org.junit.Test;
/**
- * Created by shaoshi on 1/30/15.
*/
public class IIDescManagerTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
index c70a207..7fc7bfa 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
@@ -32,7 +32,6 @@ import java.io.IOException;
/**
- * Created by shaoshi on 1/30/15.
*/
public class IIDescTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
index cfdcaa0..d1df442 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
@@ -15,7 +15,6 @@ import java.io.IOException;
import java.util.List;
/**
- * Created by shaoshi on 2/5/15.
*/
public class IIInstanceTest extends LocalFileMetadataTestCase {
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
index e75dc51..1e065b9 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
@@ -9,7 +9,6 @@ import com.ning.compress.lzf.LZFDecoder;
import com.ning.compress.lzf.LZFEncoder;
/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
*/
public class LZFTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/Scheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/Scheduler.java b/job/src/main/java/org/apache/kylin/job/Scheduler.java
index 120b86f..592b274 100644
--- a/job/src/main/java/org/apache/kylin/job/Scheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/Scheduler.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.exception.SchedulerException;
import org.apache.kylin.job.execution.Executable;
/**
- * Created by qianzhou on 12/15/14.
*/
public interface Scheduler<T extends Executable> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java b/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
index 82da8e2..29b5324 100644
--- a/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
+++ b/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.cmd;
/**
- * Created by qianzhou on 12/4/14.
*/
public abstract class BaseCommandOutput implements ICommandOutput {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
index 371e078..2da5b2a 100644
--- a/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
@@ -34,7 +34,6 @@ import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.job.execution.AbstractExecutable;
/**
- * Created by qianzhou on 12/26/14.
*/
public class HadoopShellExecutable extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java b/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
index 9cad096..32d6ec2 100644
--- a/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
@@ -35,7 +35,6 @@ import org.apache.kylin.common.util.HiveClient;
import org.apache.kylin.common.util.JsonUtil;
/**
- * Created by qianzhou on 1/15/15.
*/
public class HqlExecutable extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
index 10c6eba..af6cb9c 100644
--- a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
@@ -48,7 +48,6 @@ import org.apache.kylin.job.tools.HadoopStatusChecker;
import com.google.common.base.Preconditions;
/**
- * Created by qianzhou on 12/25/14.
*/
public class MapReduceExecutable extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 47a3145..aa51d83 100644
--- a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -34,7 +34,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 12/26/14.
*/
public class ShellExecutable extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 6c27cd7..1741692 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.constant;
/**
- * Created by qianzhou on 1/5/15.
*/
public final class ExecutableConstants {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
index bf6ae11..4b45b94 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
@@ -34,7 +34,6 @@ import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
/**
- * Created by qianzhou on 12/25/14.
*/
public class CubingJob extends DefaultChainedExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index aca62e8..1c34441 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -46,7 +46,6 @@ import java.util.List;
import java.util.TimeZone;
/**
- * Created by qianzhou on 12/25/14.
*/
public final class CubingJobBuilder extends AbstractJobBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index 5aa6570..e9c2a2c 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -34,7 +34,6 @@ import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
/**
- * Created by qianzhou on 1/4/15.
*/
public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
index bb5abdc..ea52c36 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
@@ -38,7 +38,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 1/7/15.
*/
public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 305d570..d4983e2 100644
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -35,7 +35,6 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by qianzhou on 12/15/14.
*/
public class ExecutableDao {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
index 73436db..4dacd8a 100644
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
+++ b/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
@@ -26,7 +26,6 @@ import com.google.common.collect.Maps;
import org.apache.kylin.common.persistence.RootPersistentEntity;
/**
- * Created by qianzhou on 12/15/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ExecutableOutputPO extends RootPersistentEntity {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
index cd81d64..6a17b29 100644
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
+++ b/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
@@ -27,7 +27,6 @@ import com.google.common.collect.Maps;
import org.apache.kylin.common.persistence.RootPersistentEntity;
/**
- * Created by qianzhou on 12/15/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ExecutablePO extends RootPersistentEntity {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java b/job/src/main/java/org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java
index 15ba2c4..1aeb50f 100644
--- a/job/src/main/java/org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/deployment/HbaseConfigPrinterCLI.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.kylin.job.tools.LZOSupportnessChecker;
/**
- * Created by honma on 9/30/14.
* <p/>
* This class is assumed to be run by
* "hbase org.apache.hadoop.util.RunJar kylin-job-0.5.7-SNAPSHOT-job.jar org.apache.kylin.job.deployment.HadoopConfigPrinter "
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java b/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
index 261ad5f..8544fff 100644
--- a/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
+++ b/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.exception;
/**
- * Created by qianzhou on 12/15/14.
*/
public class ExecuteException extends Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java b/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
index e6c0e7f..f19b0ca 100644
--- a/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
+++ b/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.exception;
/**
- * Created by qianzhou on 12/26/14.
*/
public class IllegalStateTranferException extends RuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/exception/LockException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/LockException.java b/job/src/main/java/org/apache/kylin/job/exception/LockException.java
index abe2232..cf43ac9 100644
--- a/job/src/main/java/org/apache/kylin/job/exception/LockException.java
+++ b/job/src/main/java/org/apache/kylin/job/exception/LockException.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.exception;
/**
- * Created by qianzhou on 12/17/14.
*/
public class LockException extends Exception {
private static final long serialVersionUID = 2072745879281754945L;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java b/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
index 0440411..8507a53 100644
--- a/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
+++ b/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.exception;
/**
- * Created by qianzhou on 12/15/14.
*/
public class PersistentException extends Exception {
private static final long serialVersionUID = -4239863858506718998L;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java b/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
index b5bfeec..057bd4a 100644
--- a/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
+++ b/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.exception;
/**
- * Created by qianzhou on 12/15/14.
*/
public class SchedulerException extends Exception {
private static final long serialVersionUID = 349041244824274861L;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index ee1607a..54f05bf 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -41,7 +41,6 @@ import java.util.Map;
import java.util.UUID;
/**
- * Created by qianzhou on 12/16/14.
*/
public abstract class AbstractExecutable implements Executable, Idempotent {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
index 3eddf1d..f352ead 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.execution;
import java.util.List;
/**
- * Created by qianzhou on 12/15/14.
*/
public interface ChainedExecutable extends Executable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index e6b0d8e..2e7bcde 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -28,7 +28,6 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.manager.ExecutableManager;
/**
- * Created by qianzhou on 12/16/14.
*/
public class DefaultChainedExecutable extends AbstractExecutable implements ChainedExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java b/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
index d6d2ddb..6bc3281 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
@@ -23,7 +23,6 @@ import org.apache.commons.lang3.StringUtils;
import java.util.Map;
/**
- * Created by qianzhou on 1/6/15.
*/
public class DefaultOutput implements Output {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/Executable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Executable.java b/job/src/main/java/org/apache/kylin/job/execution/Executable.java
index 7bd2351..a7f1358 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/Executable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/Executable.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.exception.ExecuteException;
import java.util.Map;
/**
- * Created by qianzhou on 12/15/14.
*/
public interface Executable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java b/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
index d12bd7b..e3f99ca 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.execution;
import org.apache.kylin.common.KylinConfig;
/**
- * Created by qianzhou on 12/15/14.
*/
public interface ExecutableContext {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java b/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
index 24d23ee..5dad4b3 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
@@ -28,7 +28,6 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
/**
- * Created by qianzhou on 12/15/14.
*/
public enum ExecutableState {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java b/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index bc0137d..cddc0f7 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.execution;
import com.google.common.base.Preconditions;
/**
- * Created by qianzhou on 12/15/14.
*/
public final class ExecuteResult {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java b/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
index d6b5476..98c950e 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.execution;
import org.apache.kylin.job.exception.ExecuteException;
/**
- * Created by qianzhou on 12/15/14.
*/
public interface Idempotent {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/execution/Output.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Output.java b/job/src/main/java/org/apache/kylin/job/execution/Output.java
index a835404..4d93132 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/Output.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/Output.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.execution;
import java.util.Map;
/**
- * Created by qianzhou on 1/6/15.
*/
public interface Output {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
index 91fdc71..3081b7d 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
@@ -31,7 +31,6 @@ import java.util.Arrays;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 3/27/15.
*/
public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, Text, Text> {
protected static final Logger logger = LoggerFactory.getLogger(HiveToBaseCuboidMapper.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
index d0edbc6..6bd25a8 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 3/26/15.
*/
public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, LongWritable, Text> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java
index 692364e..cccc995 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/IGTRecordWriter.java
@@ -5,7 +5,6 @@ import org.apache.kylin.storage.gridtable.GTRecord;
import java.io.IOException;
/**
- * Created by shaoshi on 4/7/15.
*/
public interface IGTRecordWriter {
void write(Long cuboidId, GTRecord record) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index f869caa..83e4a79 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -64,7 +64,6 @@ import java.util.*;
import java.util.concurrent.BlockingQueue;
/**
- * Created by shaoshi on 3/12/2015.
*/
@SuppressWarnings("rawtypes")
public class InMemCubeBuilder implements Runnable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java
index f442264..4454e43 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidMapper.java
@@ -28,7 +28,6 @@ import java.util.Map;
import java.util.concurrent.*;
/**
- * Created by shaoshi on 3/24/15.
*/
public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, ImmutableBytesWritable, Text> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
index a287c19..7e024bc 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
@@ -25,7 +25,6 @@ import java.nio.ByteBuffer;
import java.util.List;
/**
- * Created by shaoshi on 3/25/15.
*/
public class InMemCuboidReducer extends KylinReducer<ImmutableBytesWritable, Text, ImmutableBytesWritable, KeyValue> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
index df3f345..299143f 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
@@ -18,7 +18,6 @@ import java.nio.ByteBuffer;
import java.util.BitSet;
/**
- * Created by shaoshi on 4/7/15.
*/
public class MapContextGTRecordWriter implements IGTRecordWriter {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
index 9e5bd92..c25b164 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
@@ -37,7 +37,6 @@ import org.apache.kylin.job.JobInstance;
import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
*/
public class IIFlattenHiveJob extends AbstractHadoopJob {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
index b2dc06d..e959ae2 100644
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
+++ b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentMap;
/**
- * Created by qianzhou on 12/16/14.
*/
public class DefaultContext implements ExecutableContext {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index b486a3e..2acba4e 100644
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -54,7 +54,6 @@ import com.google.common.collect.Maps;
import org.apache.kylin.common.util.HadoopUtil;
/**
- * Created by qianzhou on 12/15/14.
*/
public class DefaultScheduler implements Scheduler<AbstractExecutable>, ConnectionStateListener {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
index 7382bbf..4b93b6e 100644
--- a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
+++ b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job.invertedindex;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
/**
- * Created by shaoshi on 1/15/15.
*/
public class IIJob extends DefaultChainedExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
index e2bcfd2..26a67f4 100644
--- a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
@@ -42,7 +42,6 @@ import org.apache.kylin.job.hadoop.invertedindex.InvertedIndexJob;
import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
/**
- * Created by shaoshi on 1/15/15.
*/
public final class IIJobBuilder extends AbstractJobBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index f51ff69..3c066cb 100644
--- a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -46,7 +46,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
- * Created by qianzhou on 12/16/14.
*/
public class ExecutableManager {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
index 5577dce..d1be49a 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
@@ -14,7 +14,6 @@ import java.util.List;
import java.util.Properties;
/**
- * Created by Hongbin Ma(Binmahone) on 5/5/15.
*/
public class KafkaDataLoader {
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 5211516..7854fd5 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -58,7 +58,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
- * Created by qianzhou on 3/26/15.
*/
public class StreamingBootstrap {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index ba13434..c2dd2ea 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -40,7 +40,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Created by qianzhou on 3/26/15.
*/
public class StreamingCLI {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
index 65e645d..e1ebe20 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
/**
- * Created by honma on 11/11/14.
*/
public class CleanHtableCLI extends AbstractHadoopJob {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index 97373b8..4dfbf6c 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -55,7 +55,6 @@ import java.util.List;
import java.util.Map;
/**
- * Created by honma on 9/3/14.
* <p/>
* This tool serves for the purpose of migrating cubes. e.g. upgrade cube from
* dev env to test(prod) env, or vice versa.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 2a9abf9..80ae002 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
- * Created by qianzhou on 1/20/15.
*/
public class HadoopStatusGetter {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java b/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
index 8dcfa6d..e38a574 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
@@ -19,7 +19,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone) on 2/13/15.
*/
public class HbaseStreamingInput {
private static final Logger logger = LoggerFactory.getLogger(HbaseStreamingInput.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java b/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
index 80b9fa7..94eac97 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
@@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
- * Created by honma on 11/11/14.
*/
@SuppressWarnings("static-access")
public class HtableAlterMetadataCLI extends AbstractHadoopJob {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/main/java/org/apache/kylin/job/tools/LZOSupportnessChecker.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/LZOSupportnessChecker.java b/job/src/main/java/org/apache/kylin/job/tools/LZOSupportnessChecker.java
index ae16f7c..e9eabb7 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/LZOSupportnessChecker.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/LZOSupportnessChecker.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hbase.util.CompressionTest;
import java.io.File;
/**
- * Created by honma on 10/21/14.
*/
public class LZOSupportnessChecker {
public static boolean getSupportness() {
[28/50] [abbrv] incubator-kylin git commit: KYLIN-754 add check
supportness before enabling lzo
Posted by sh...@apache.org.
KYLIN-754 add check supportness before enabling lzo
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a2906ae1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a2906ae1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a2906ae1
Branch: refs/heads/streaming-localdict
Commit: a2906ae12a17d78197fc0e2d2ddc80d708c36d24
Parents: 93847c3
Author: honma <ho...@ebay.com>
Authored: Fri May 8 10:03:22 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri May 8 10:03:22 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/hadoop/hbase/CreateHTableJob.java | 4 ++--
.../kylin/job/hadoop/invertedindex/IICreateHTableJob.java | 10 +++++++++-
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2906ae1/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index 69d241f..cbe8c10 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -119,10 +119,10 @@ public class CreateHTableJob extends AbstractHadoopJob {
cf.setMaxVersions(1);
if (LZOSupportnessChecker.getSupportness()) {
- logger.info("hbase will use lzo to compress data");
+ logger.info("hbase will use lzo to compress cube data");
cf.setCompressionType(Algorithm.LZO);
} else {
- logger.info("hbase will not use lzo to compress data");
+ logger.info("hbase will not use lzo to compress cube data");
}
cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2906ae1/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index cf8d005..23a4930 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -39,6 +39,7 @@ import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.job.tools.DeployCoprocessorCLI;
+import org.apache.kylin.job.tools.LZOSupportnessChecker;
import org.apache.kylin.metadata.realization.IRealizationConstants;
/**
@@ -66,7 +67,14 @@ public class IICreateHTableJob extends AbstractHadoopJob {
HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor cf = new HColumnDescriptor(IIDesc.HBASE_FAMILY);
cf.setMaxVersions(1);
- cf.setCompressionType(Compression.Algorithm.LZO);
+
+ if (LZOSupportnessChecker.getSupportness()) {
+ logger.info("hbase will use lzo to compress II data");
+ cf.setCompressionType(Compression.Algorithm.LZO);
+ } else {
+ logger.info("hbase will not use lzo to compress II data");
+ }
+
cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
tableDesc.addFamily(cf);
tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());
[17/50] [abbrv] incubator-kylin git commit: KYLIN-749 remove creating
htable on each streamcli start
Posted by sh...@apache.org.
KYLIN-749 remove creating htable on each streamcli start
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f9b1442e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f9b1442e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f9b1442e
Branch: refs/heads/streaming-localdict
Commit: f9b1442e58149a9a16cb8192f445418687913715
Parents: 35f803c
Author: honma <ho...@ebay.com>
Authored: Wed May 6 18:01:59 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 18:01:59 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/streaming/StreamingBootstrap.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f9b1442e/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 1f7608b..88302c8 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -41,10 +41,10 @@ import kafka.cluster.Broker;
import kafka.javaapi.PartitionMetadata;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.streaming.*;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
@@ -137,7 +137,10 @@ public class StreamingBootstrap {
streamingOffset = Math.max(streamingOffset, earliestOffset);
logger.info("starting offset is " + streamingOffset);
- IICreateHTableJob.main(new String[] { "-iiname", kafkaConfig.getIiName(), "-htablename", iiSegment.getStorageLocationIdentifier() });
+ if (!HBaseConnection.tableExists(kylinConfig.getStorageUrl(), iiSegment.getStorageLocationIdentifier())) {
+ logger.error("no htable:" + iiSegment.getStorageLocationIdentifier() + " found");
+ throw new IllegalStateException("please create htable:" + iiSegment.getStorageLocationIdentifier() + " first");
+ }
KafkaConsumer consumer = new KafkaConsumer(kafkaConfig.getTopic(), partitionId, streamingOffset, kafkaConfig.getBrokers(), kafkaConfig, parallelism);
kafkaConsumers.put(getKey(streaming, partitionId), consumer);
[45/50] [abbrv] incubator-kylin git commit: KYLIN-759 make storage
cache realization independent
Posted by sh...@apache.org.
KYLIN-759 make storage cache realization independent
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/91792351
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/91792351
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/91792351
Branch: refs/heads/streaming-localdict
Commit: 91792351312dee2c54c6c3b0f861e35722caa9df
Parents: ac515a7
Author: honma <ho...@ebay.com>
Authored: Thu May 14 17:31:19 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 14 20:22:04 2015 +0800
----------------------------------------------------------------------
.../AbstractCacheFledgedStorageEngine.java | 23 +++++----
.../cache/CacheFledgedDynamicStorageEngine.java | 6 +--
.../cache/CacheFledgedStaticStorageEngine.java | 4 +-
.../apache/kylin/storage/cache/EhcacheTest.java | 51 +++++++++++---------
4 files changed, 47 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91792351/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
index 09f0026..8e1b50a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
@@ -3,10 +3,9 @@ package org.apache.kylin.storage.cache;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.Configuration;
import net.sf.ehcache.config.PersistenceConfiguration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
-
import org.apache.kylin.metadata.realization.StreamSQLDigest;
import org.apache.kylin.metadata.tuple.TeeTupleItrListener;
import org.apache.kylin.storage.ICachableStorageEngine;
@@ -19,7 +18,13 @@ import org.slf4j.LoggerFactory;
*/
public abstract class AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFledgedStorageEngine.class);
- protected static CacheManager cacheManager = CacheManager.create();
+ protected static CacheManager cacheManager;
+
+ static {
+ Configuration conf = new Configuration();
+ conf.setMaxBytesLocalHeap("1024M");
+ cacheManager = CacheManager.create(conf);
+ }
protected final ICachableStorageEngine underlyingStorage;
protected StreamSQLDigest streamSQLDigest;
@@ -28,20 +33,20 @@ public abstract class AbstractCacheFledgedStorageEngine implements IStorageEngin
public AbstractCacheFledgedStorageEngine(ICachableStorageEngine underlyingStorage) {
this.underlyingStorage = underlyingStorage;
this.queryCacheExists = false;
- this.makeCacheIfNecessary(underlyingStorage.getClass().getName());
+ this.makeCacheIfNecessary(underlyingStorage.getStorageUUID());
}
- private void makeCacheIfNecessary(String storageClassName) {
- if (cacheManager.getCache(storageClassName) == null) {
- logger.info("Cache for {} initting:", storageClassName);
+ private void makeCacheIfNecessary(String realizationUUID) {
+ if (cacheManager.getCache(realizationUUID) == null) {
+ logger.info("Cache for {} initting...", realizationUUID);
// TODO: L4J [2015-04-20 10:44:03,817][WARN][net.sf.ehcache.pool.sizeof.ObjectGraphWalker] - The configured limit of 1,000 object references was reached while attempting to calculate the size of the object graph. Severe performance degradation could occur if the sizing operation continues. This can be avoided by setting the CacheManger or Cache <sizeOfPolicy> elements maxDepthExceededBehavior to "abort" or adding stop points with @IgnoreSizeOf annotations. If performance degradation is NOT an issue at the configured limit, raise the limit value using the CacheManager or Cache <sizeOfPolicy
//Create a Cache specifying its configuration.
- Cache storageCache = new Cache(new CacheConfiguration(storageClassName, 0).//
+ Cache storageCache = new Cache(new CacheConfiguration(realizationUUID, 0).//
memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).//
eternal(false).//
timeToIdleSeconds(86400).//
diskExpiryThreadIntervalSeconds(0).//
- maxBytesLocalHeap(256, MemoryUnit.MEGABYTES).//
+ //maxBytesLocalHeap(256, MemoryUnit.MEGABYTES).//already defined at manager scope
persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
//TODO: deal with failed queries, and only cache too long query
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91792351/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
index 0a4fd2a..f62551b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
@@ -36,7 +36,7 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
this.partitionColRef = partitionColRef;
Preconditions.checkArgument(this.partitionColRef != null, "For dynamic columns like " + //
- this.underlyingStorage.getClass().getName() + ", partition column must be provided");
+ this.underlyingStorage.getStorageUUID()+ ", partition column must be provided");
}
@Override
@@ -47,7 +47,7 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
StreamSQLResult cachedResult = null;
- Cache cache = cacheManager.getCache(this.underlyingStorage.getClass().getName());
+ Cache cache = cacheManager.getCache(this.underlyingStorage.getStorageUUID());
Element element = cache.get(streamSQLDigest);
if (element != null) {
this.queryCacheExists = true;
@@ -133,7 +133,7 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
}
StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, ts, partitionColRef);
- cacheManager.getCache(this.underlyingStorage.getClass().getName()).put(new Element(streamSQLDigest, newCacheEntry));
+ cacheManager.getCache(this.underlyingStorage.getStorageUUID()).put(new Element(streamSQLDigest, newCacheEntry));
logger.info("cache after the query: " + newCacheEntry);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91792351/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
index a309c24..72372c6 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
@@ -32,7 +32,7 @@ public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorage
streamSQLDigest = new StreamSQLDigest(sqlDigest, null);
StreamSQLResult cachedResult = null;
- Cache cache = cacheManager.getCache(this.underlyingStorage.getClass().getName());
+ Cache cache = cacheManager.getCache(this.underlyingStorage.getStorageUUID());
Element element = cache.get(streamSQLDigest);
if (element != null) {
this.queryCacheExists = true;
@@ -66,7 +66,7 @@ public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorage
@Override
public void notify(List<ITuple> duplicated) {
StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, Ranges.<Long> all(), null);
- cacheManager.getCache(this.underlyingStorage.getClass().getName()).put(new Element(streamSQLDigest, newCacheEntry));
+ cacheManager.getCache(this.underlyingStorage.getStorageUUID()).put(new Element(streamSQLDigest, newCacheEntry));
logger.info("cache after the query: " + newCacheEntry);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91792351/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
index 1dec9e6..fe390de 100644
--- a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -2,8 +2,9 @@ package org.apache.kylin.storage.cache;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.Configuration;
import net.sf.ehcache.config.PersistenceConfiguration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
@@ -14,7 +15,11 @@ import org.junit.Test;
public class EhcacheTest {
@Test
public void basicTest() throws InterruptedException {
- CacheManager cacheManager = CacheManager.create();
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+
+ Configuration conf = new Configuration();
+ conf.setMaxBytesLocalHeap("100M");
+ CacheManager cacheManager = CacheManager.create(conf);
//Create a Cache specifying its configuration.
Cache testCache = //Create a Cache specifying its configuration.
@@ -23,31 +28,31 @@ public class EhcacheTest {
eternal(false).//
timeToIdleSeconds(86400).//
diskExpiryThreadIntervalSeconds(0).//
- maxBytesLocalHeap(100, MemoryUnit.MEGABYTES).//
+ //maxBytesLocalHeap(1000, MemoryUnit.MEGABYTES).//
persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
cacheManager.addCache(testCache);
- //
- // byte[] blob2 = new byte[(1024 * 400 * 1024)];//400M
- //
- // testCache.put(new Element("1", blob));
- // System.out.println(testCache.get("1") == null);
- // System.out.println(testCache.getSize());
- // System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
- // System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
- // testCache.put(new Element("2", blob));
- // System.out.println(testCache.get("1") == null);
- // System.out.println(testCache.getSize());
- // System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
- // System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
- // testCache.put(new Element("3", blob));
- // System.out.println(testCache.get("1") == null);
- // System.out.println(testCache.get("2") == null);
- // System.out.println(testCache.get("3") == null);
- // System.out.println(testCache.getSize());
- // System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
- // System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ byte[] blob = new byte[(1024 * 40 * 1024)];//400M
+
+ testCache.put(new Element("1", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ testCache.put(new Element("2", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ testCache.put(new Element("3", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.get("2") == null);
+ System.out.println(testCache.get("3") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
cacheManager.shutdown();
}
[15/50] [abbrv] incubator-kylin git commit: KYLIN-749 add stream data
generator
Posted by sh...@apache.org.
KYLIN-749 add stream data generator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/eeb2abe0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/eeb2abe0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/eeb2abe0
Branch: refs/heads/streaming-localdict
Commit: eeb2abe09d35e594e7e0e1cac14475c4c5f234d5
Parents: f256263
Author: honma <ho...@ebay.com>
Authored: Tue May 5 14:38:31 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:16:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/TimeUtil.java | 17 +++++
.../apache/kylin/common/util/TimeUtilTest.java | 44 +++++++++++
.../kylin/job/streaming/StreamingBootstrap.java | 36 +++++----
.../java/org/apache/kylin/job/DataGenTest.java | 4 +
.../job/dataGen/StreamingDataGenerator.java | 78 ++++++++++++++++++++
5 files changed, 160 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eeb2abe0/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
new file mode 100644
index 0000000..01fc2c1
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -0,0 +1,17 @@
+package org.apache.kylin.common.util;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/4/15.
+ */
+public class TimeUtil {
+ private static int ONE_MINUTE_TS = 60 * 1000;
+ private static int ONE_HOUR_TS = 60 * 60 * 1000;
+
+ public static long getMinuteStart(long ts) {
+ return ts / ONE_MINUTE_TS * ONE_MINUTE_TS;
+ }
+
+ public static long getHourStart(long ts) {
+ return ts / ONE_HOUR_TS * ONE_HOUR_TS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eeb2abe0/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
new file mode 100644
index 0000000..1442412
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -0,0 +1,44 @@
+package org.apache.kylin.common.util;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/4/15.
+ */
+public class TimeUtilTest {
+ public static long normalizeTime(long timeMillis, NormalizeUnit unit) {
+ Calendar a = Calendar.getInstance();
+ Calendar b = Calendar.getInstance();
+ b.clear();
+
+ a.setTimeInMillis(timeMillis);
+ if (unit == NormalizeUnit.MINUTE) {
+ b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE));
+ } else if (unit == NormalizeUnit.HOUR) {
+ b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0);
+ }
+ return b.getTimeInMillis();
+ }
+
+ @Test
+ public void basicTest() throws ParseException {
+ java.text.DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+
+ long t1 = dateFormat.parse("2012/01/01 00:00:01").getTime();
+ Assert.assertEquals(normalizeTime(t1, NormalizeUnit.HOUR), TimeUtil.getHourStart(t1));
+ Assert.assertEquals(normalizeTime(t1, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t1));
+
+ long t2 = dateFormat.parse("2012/12/31 11:02:01").getTime();
+ Assert.assertEquals(normalizeTime(t2, NormalizeUnit.HOUR), TimeUtil.getHourStart(t2));
+ Assert.assertEquals(normalizeTime(t2, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t2));
+ }
+
+ public enum NormalizeUnit {
+ MINUTE, HOUR
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eeb2abe0/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 3e8f3f4..1688fc2 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,27 +34,30 @@
package org.apache.kylin.job.streaming;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
import kafka.javaapi.PartitionMetadata;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.streaming.*;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Constructor;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
/**
* Created by qianzhou on 3/26/15.
@@ -69,6 +72,12 @@ public class StreamingBootstrap {
private Map<String, KafkaConsumer> kafkaConsumers = Maps.newConcurrentMap();
+ private StreamingBootstrap(KylinConfig kylinConfig) {
+ this.kylinConfig = kylinConfig;
+ this.streamingManager = StreamingManager.getInstance(kylinConfig);
+ this.iiManager = IIManager.getInstance(kylinConfig);
+ }
+
public static StreamingBootstrap getInstance(KylinConfig kylinConfig) {
final StreamingBootstrap bootstrap = new StreamingBootstrap(kylinConfig);
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@@ -80,12 +89,6 @@ public class StreamingBootstrap {
return bootstrap;
}
- private StreamingBootstrap(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- this.streamingManager = StreamingManager.getInstance(kylinConfig);
- this.iiManager = IIManager.getInstance(kylinConfig);
- }
-
private static Broker getLeadBroker(KafkaConfig kafkaConfig, int partitionId) {
final PartitionMetadata partitionMetadata = KafkaRequester.getPartitionMetadata(kafkaConfig.getTopic(), partitionId, kafkaConfig.getBrokers(), kafkaConfig);
if (partitionMetadata != null && partitionMetadata.errorCode() == 0) {
@@ -137,11 +140,7 @@ public class StreamingBootstrap {
streamingOffset = Math.max(streamingOffset, earliestOffset);
logger.info("starting offset is " + streamingOffset);
- if (!HBaseConnection.tableExists(kylinConfig.getStorageUrl(), iiSegment.getStorageLocationIdentifier())) {
- logger.error("no htable:" + iiSegment.getStorageLocationIdentifier() + " found");
- throw new IllegalStateException("please create htable:" + iiSegment.getStorageLocationIdentifier() + " first");
- }
-
+ IICreateHTableJob.main(new String[] { "-iiname", "nous_ii", "-htablename", "KYLIN_2SKJ8JNOUS" });
KafkaConsumer consumer = new KafkaConsumer(kafkaConfig.getTopic(), partitionId, streamingOffset, kafkaConfig.getBrokers(), kafkaConfig, parallelism);
kafkaConsumers.put(getKey(streaming, partitionId), consumer);
@@ -166,7 +165,6 @@ public class StreamingBootstrap {
}
}
-
}
private String getKey(String streaming, int partitionId) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eeb2abe0/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index 89e964e..c05ddc7 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -54,4 +54,8 @@ public class DataGenTest extends LocalFileMetadataTestCase {
DeployUtil.overrideFactTableData(content, "default.test_kylin_fact");
}
+ @Test
+ public void testStreaming() throws Exception {
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eeb2abe0/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
new file mode 100644
index 0000000..d822a9a
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
@@ -0,0 +1,78 @@
+package org.apache.kylin.job.dataGen;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.TimeUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class StreamingDataGenerator {
+ private static final Logger logger = LoggerFactory.getLogger(StreamingDataGenerator.class);
+ private static Random random = new Random();
+ private static String[] decimalFormat = new String[] { "%.4f", "%.5f", "%.6f" };
+
+ public static Iterator<List<String>> generate(final long start, final long end, final int count) {
+ final KylinConfig config = KylinConfig.getInstanceFromEnv();
+ final IIInstance ii = IIManager.getInstance(config).getII("test_streaming_table");
+ final IIDesc iiDesc = ii.getDescriptor();
+ final MetadataManager metadataManager = MetadataManager.getInstance(config);
+ final ColumnDesc[] columnDescs = metadataManager.getTableDesc(iiDesc.getFactTableName()).getColumns();
+
+ return new Iterator<List<String>>() {
+ private Map<String, String> values = Maps.newHashMap();
+
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public List<String> next() {
+ values.clear();
+ long ts = this.createTs(start, end);
+ values.put("ts", Long.toString(ts));
+ values.put("minute_start", Long.toString(TimeUtil.getMinuteStart(ts)));
+ values.put("hour_start", Long.toString(TimeUtil.getHourStart(ts)));
+ values.put("itm", Integer.toString(random.nextInt(20)));
+ values.put("site", Integer.toString(random.nextInt(5)));
+
+ values.put("gmv", String.format(decimalFormat[random.nextInt(3)], random.nextFloat() * 100));
+ values.put("item_count", Integer.toString(random.nextInt(5)));
+
+ if (values.size() != columnDescs.length) {
+ throw new RuntimeException("the structure of streaming table has changed, need to modify generator too");
+ }
+
+ List<String> ret = Lists.newArrayList();
+ for (ColumnDesc columnDesc : columnDescs) {
+ String name = columnDesc.getName();
+ ret.add(values.get(name));
+ }
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ }
+
+ private long createTs(final long start, final long end) {
+ return start + (long) (random.nextDouble() * (end - start));
+ }
+ };
+ }
+
+}
[38/50] [abbrv] incubator-kylin git commit: KYLIN-760 Improve the
hasing performance in Sampling cuboid size
Posted by sh...@apache.org.
KYLIN-760 Improve the hasing performance in Sampling cuboid size
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e1e1aea9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e1e1aea9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e1e1aea9
Branch: refs/heads/streaming-localdict
Commit: e1e1aea9f90116ba46f4399acb47dcaeecd8d14a
Parents: d939eb1
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed May 13 10:09:42 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed May 13 10:10:25 2015 +0800
----------------------------------------------------------------------
.../cube/FactDistinctHiveColumnsMapper.java | 28 +++-
.../job/hadoop/cubev2/CubeSamplingTest.java | 135 +++++++++++++++++++
2 files changed, 157 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e1e1aea9/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
index 20df4c4..a884465 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
@@ -22,11 +22,12 @@ import com.google.common.collect.Lists;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
@@ -58,6 +59,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private HashFunction hf = null;
private int rowCount = 0;
private int SAMPING_PERCENTAGE = 5;
+ private ByteArray[] row_hashcodes = null;
@Override
protected void setup(Context context) throws IOException {
@@ -83,7 +85,11 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
allCuboidsHLL[i] = new HyperLogLogPlusCounter(14);
}
- hf = Hashing.md5();
+ hf = Hashing.murmur3_32();
+ row_hashcodes = new ByteArray[nRowKey];
+ for (int i = 0; i < nRowKey; i++) {
+ row_hashcodes[i] = new ByteArray();
+ }
}
}
@@ -96,7 +102,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
int position = 0;
for (int i = 0; i < nRowKey; i++) {
if ((mask & cuboidId) > 0) {
- indice[position] = intermediateTableDesc.getRowKeyColumnIndexes()[i];
+ indice[position] = i;
position++;
}
mask = mask >> 1;
@@ -136,12 +142,22 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
}
private void putRowKeyToHLL(List<String> row) {
+
+ //generate hash for each row key column
+ for (int i = 0; i < nRowKey; i++) {
+ Hasher hc = hf.newHasher();
+ if (row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]) != null) {
+ row_hashcodes[i].set(hc.putString(row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i])).hash().asBytes());
+ } else {
+ row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
+ }
+ }
+
+ // user the row key column hash to get a consolidated hash for each cuboid
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
Hasher hc = hf.newHasher();
for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
- if (row.get(allCuboidsBitSet[i][position]) != null)
- hc.putString(row.get(allCuboidsBitSet[i][position]));
- hc.putString(",");
+ hc.putBytes(row_hashcodes[allCuboidsBitSet[i][position]].array());
}
allCuboidsHLL[i].add(hc.hash().asBytes());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e1e1aea9/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
new file mode 100644
index 0000000..f321fc3
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cubev2;
+
+import com.google.common.collect.Lists;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+/**
+ * Created by shaoshi on 5/12/15.
+ */
+public class CubeSamplingTest {
+
+ private static final int ROW_LENGTH = 10;
+
+ private final List<String> row = new ArrayList<String>(ROW_LENGTH);
+ private final ByteArray[] row_index = new ByteArray[ROW_LENGTH];
+
+ private Integer[][] allCuboidsBitSet;
+ private HashFunction hf = null;
+ private long baseCuboidId;
+ private HyperLogLogPlusCounter[] allCuboidsHLL = null;
+ private final byte[] seperator = Bytes.toBytes(",");
+
+ @Before
+ public void setup() {
+
+ baseCuboidId = (1l << ROW_LENGTH) - 1;
+ List<Long> allCuboids = Lists.newArrayList();
+ List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
+ for (long i = 1; i < baseCuboidId; i++) {
+ allCuboids.add(i);
+ addCuboidBitSet(i, allCuboidsBitSetList);
+ }
+
+ allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[allCuboidsBitSetList.size()][]);
+ System.out.println("Totally have " + allCuboidsBitSet.length + " cuboids.");
+ allCuboidsHLL = new HyperLogLogPlusCounter[allCuboids.size()];
+ for (int i = 0; i < allCuboids.size(); i++) {
+ allCuboidsHLL[i] = new HyperLogLogPlusCounter(14);
+ }
+
+ // hf = Hashing.goodFastHash(32);
+// hf = Hashing.md5();
+ hf = Hashing.murmur3_32();
+
+ for (int i = 0; i < ROW_LENGTH; i++) {
+ row_index[i] = new ByteArray();
+ }
+ }
+
+ private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) {
+ BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
+ Integer[] indice = new Integer[bitSet.cardinality()];
+
+ long mask = Long.highestOneBit(baseCuboidId);
+ int position = 0;
+ for (int i = 0; i < ROW_LENGTH; i++) {
+ if ((mask & cuboidId) > 0) {
+ indice[position] = i;
+ position++;
+ }
+ mask = mask >> 1;
+ }
+
+ allCuboidsBitSet.add(indice);
+
+ }
+
+ @Test
+ public void test() {
+
+ long start = System.currentTimeMillis();
+ List<String> row;
+ for (int i = 0; i < 10000; i++) {
+ row = getRandomRow();
+ putRowKeyToHLL(row);
+ }
+
+ long duration = System.currentTimeMillis() - start;
+ System.out.println("The test takes " + duration / 1000 + "seconds.");
+ }
+
+ private void putRowKeyToHLL(List<String> row) {
+ int x = 0;
+ for (String field : row) {
+ Hasher hc = hf.newHasher();
+ row_index[x++].set(hc.putString(field).hash().asBytes());
+ }
+
+ for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
+ Hasher hc = hf.newHasher();
+ for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
+ hc.putBytes(row_index[allCuboidsBitSet[i][position]].array());
+ hc.putBytes(seperator);
+ }
+ allCuboidsHLL[i].add(hc.hash().asBytes());
+ }
+ }
+
+ private List<String> getRandomRow() {
+ row.clear();
+ for (int i = 0; i < ROW_LENGTH; i++) {
+ row.add(RandomStringUtils.random(10));
+ }
+ return row;
+ }
+}
[27/50] [abbrv] incubator-kylin git commit: KYLIN-607 Use new cubing
in BuildCubeWithEngineTest
Posted by sh...@apache.org.
KYLIN-607 Use new cubing in BuildCubeWithEngineTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/93847c3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/93847c3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/93847c3f
Branch: refs/heads/streaming-localdict
Commit: 93847c3fc904923132023b98e5e62b9ab4d4ebcb
Parents: 8f65c1c
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu May 7 17:30:58 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu May 7 17:30:58 2015 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/kylin.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/93847c3f/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 1830eaf..596e811 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -34,7 +34,7 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
kylin.job.concurrent.max.limit=10
# Whether calculate cube in mem in each mapper;
-kylin.job.cubing.inMem=false
+kylin.job.cubing.inMem=true
#the percentage of the sampling, default 25%
kylin.job.cubing.inMem.sampling.percent=25
[41/50] [abbrv] incubator-kylin git commit: KYLIN-763 remove author
information
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/StreamingManagerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/StreamingManagerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/StreamingManagerTest.java
index b58dbfd..ea76a93 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/StreamingManagerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/StreamingManagerTest.java
@@ -45,7 +45,6 @@ import static junit.framework.TestCase.assertNotNull;
import static org.junit.Assert.fail;
/**
- * Created by qianzhou on 3/25/15.
*/
public class StreamingManagerTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/invertedindex/PrintOutStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/invertedindex/PrintOutStreamBuilder.java b/streaming/src/test/java/org/apache/kylin/streaming/invertedindex/PrintOutStreamBuilder.java
index 3b6ae02..b3d7742 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/invertedindex/PrintOutStreamBuilder.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/invertedindex/PrintOutStreamBuilder.java
@@ -44,7 +44,6 @@ import java.util.List;
import java.util.concurrent.BlockingQueue;
/**
- * Created by qianzhou on 3/25/15.
*/
public class PrintOutStreamBuilder extends StreamBuilder {
[37/50] [abbrv] incubator-kylin git commit: Merge pull request #461
from qhzhou/new_igt_store
Posted by sh...@apache.org.
Merge pull request #461 from qhzhou/new_igt_store
KYLIN-728 use more accurate memory estimation to reduce disk io, acce…
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d939eb1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d939eb1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d939eb1e
Branch: refs/heads/streaming-localdict
Commit: d939eb1e33dd97ed407765426a9aac00228044c2
Parents: 033f2ae 9e58ae8
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue May 12 16:47:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue May 12 16:47:16 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cubev2/InMemCubeBuilder.java | 40 +++++++++-----------
.../kylin/storage/gridtable/GTComboStore.java | 2 -
.../storage/gridtable/diskstore/FileSystem.java | 2 +
.../gridtable/diskstore/GTDiskStore.java | 9 ++++-
.../gridtable/diskstore/HadoopFileSystem.java | 14 ++++++-
.../gridtable/diskstore/LocalFileSystem.java | 12 +++++-
6 files changed, 48 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] incubator-kylin git commit: KYLIN-749 bug fix
Posted by sh...@apache.org.
KYLIN-749 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/29da9ece
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/29da9ece
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/29da9ece
Branch: refs/heads/streaming-localdict
Commit: 29da9ece5b55c21a1f880cb66cfff0348b529c3f
Parents: c8bc428
Author: honma <ho...@ebay.com>
Authored: Tue May 5 18:00:15 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:16:48 2015 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamingBootstrap.java | 19 ++++++++-----------
.../java/org/apache/kylin/job/DataGenTest.java | 4 +++-
2 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29da9ece/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 1688fc2..1f7608b 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,19 +34,13 @@
package org.apache.kylin.job.streaming;
-import java.lang.reflect.Constructor;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
import kafka.javaapi.PartitionMetadata;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
@@ -56,8 +50,11 @@ import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
/**
* Created by qianzhou on 3/26/15.
@@ -140,7 +137,7 @@ public class StreamingBootstrap {
streamingOffset = Math.max(streamingOffset, earliestOffset);
logger.info("starting offset is " + streamingOffset);
- IICreateHTableJob.main(new String[] { "-iiname", "nous_ii", "-htablename", "KYLIN_2SKJ8JNOUS" });
+ IICreateHTableJob.main(new String[] { "-iiname", kafkaConfig.getIiName(), "-htablename", iiSegment.getStorageLocationIdentifier() });
KafkaConsumer consumer = new KafkaConsumer(kafkaConfig.getTopic(), partitionId, streamingOffset, kafkaConfig.getBrokers(), kafkaConfig, parallelism);
kafkaConsumers.put(getKey(streaming, partitionId), consumer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29da9ece/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index 4e3cf17..7b10a22 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -72,7 +72,7 @@ public class DataGenTest extends LocalFileMetadataTestCase {
int totalCount = 10000;
int counter = 0;
- Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-01-06"), totalCount);
+ Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-02-05"), totalCount);
iterator = SortUtil.extractAndSort(iterator, new Function<String, Comparable>() {
public Comparable apply(String input) {
@@ -80,6 +80,8 @@ public class DataGenTest extends LocalFileMetadataTestCase {
}
});
+ //FileUtils.writeLines(new File("//Users/honma/streaming_table_data"),Lists.newArrayList(iterator));
+
long lastTs = 0;
while (iterator.hasNext()) {
counter++;
[20/50] [abbrv] incubator-kylin git commit: KYLIN-749 added
TimeStrDictionary for non date types.
Posted by sh...@apache.org.
KYLIN-749 added TimeStrDictionary for non date types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9854f7d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9854f7d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9854f7d3
Branch: refs/heads/streaming-localdict
Commit: 9854f7d3cdc88e7af146e97956a165108547a041
Parents: 05e0bad
Author: honma <ho...@ebay.com>
Authored: Thu May 7 15:00:56 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:04:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 6 +-
.../java/org/apache/kylin/dict/Dictionary.java | 9 +-
.../apache/kylin/dict/DictionaryGenerator.java | 4 +-
.../apache/kylin/dict/DictionaryManager.java | 4 +-
.../apache/kylin/dict/TimeStrDictionary.java | 115 +++++++++++++++++++
.../kylin/dict/TimeStrDictionaryTests.java | 61 ++++++++++
.../invertedindex/index/TableRecordInfo.java | 6 +-
.../invertedindex/model/IIKeyValueCodec.java | 2 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 4 +-
.../job/hadoop/cube/MergeCuboidMapperTest.java | 2 +-
.../metadata/filter/ColumnTupleFilter.java | 14 +--
.../metadata/filter/CompareTupleFilter.java | 19 +--
.../metadata/filter/ConstantTupleFilter.java | 7 +-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../kylin/metadata/project/ProjectL2Cache.java | 4 +-
.../metadata/realization/SQLDigestUtil.java | 11 +-
.../kylin/query/relnode/OLAPFilterRel.java | 10 +-
.../apache/kylin/query/test/KylinQueryTest.java | 2 +-
.../storage/gridtable/GTFilterScanner.java | 2 +-
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../storage/gridtable/GTInvertedIndex.java | 2 +-
.../storage/gridtable/GTScanRangePlanner.java | 2 +-
.../kylin/storage/gridtable/GTScanRequest.java | 4 +-
.../apache/kylin/storage/gridtable/GTUtil.java | 2 +-
.../kylin/storage/hbase/CubeTupleConverter.java | 2 +-
.../storage/hbase/DerivedFilterTranslator.java | 4 +-
.../hbase/coprocessor/CoprocessorFilter.java | 4 +-
.../DateConditionInplaceModifier.java | 63 ++++++++++
.../endpoint/EndpointTupleIterator.java | 21 ++--
.../org/apache/kylin/storage/tuple/Tuple.java | 14 +--
.../DateConditionInplaceModifierTest.java | 34 ++++++
.../apache/kylin/storage/test/StorageTest.java | 2 +-
32 files changed, 359 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index db80934..efa1459 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -63,8 +63,9 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
- System.out.println(Long.parseLong("0032"));
- System.out.println(time(1000L * Integer.MAX_VALUE));
+ System.out.println(time(946684800000L));
+
+ System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1000L * Integer.MAX_VALUE));
System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(3, 10)));
@@ -80,7 +81,6 @@ public class BasicTest {
System.out.println("n");
System.out.println("The start key is set to " + null);
- System.out.println(time(946684800000L));
long current = System.currentTimeMillis();
System.out.println(time(current));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index d6addb5..070e9de 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -18,13 +18,12 @@
package org.apache.kylin.dict;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-
import org.apache.hadoop.io.Writable;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
/**
* A bi-way dictionary that maps from dimension/column values to IDs and vice
* versa. By storing IDs instead of real values, the size of cube is
@@ -45,7 +44,7 @@ abstract public class Dictionary<T> implements Writable {
public static final byte NULL = (byte) 0xff;
// ID with all bit-1 (0xff e.g.) reserved for NULL value
- public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff };
+ public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
abstract public int getMinId();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 59d7bb3..467456a 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -147,8 +147,8 @@ public class DictionaryGenerator {
}
}
- //FIXME: except for date type, all other date time family types are treated as string
- return buildStringDict(values, baseId, nSamples, samples);
+ //FIXME: except for date type, all other date time family types are treated as TimeStrDictionary
+ return new TimeStrDictionary();
}
private static Dictionary buildStringDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 8e5d599..fdedaf3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -194,7 +194,7 @@ public class DictionaryManager {
if (model == null) {
srcTable = col.getTable();
srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
+ srcColIdx = col.getColumnDesc().getZeroBasedIndex();
int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount();
table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns);
return new Object[] { srcTable, srcCol, srcColIdx, table };
@@ -220,7 +220,7 @@ public class DictionaryManager {
}
srcTable = col.getTable();
srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
+ srcColIdx = col.getColumnDesc().getZeroBasedIndex();
if (model.isFactTable(col.getTable())) {
table = new FileTable(factColumnsPath + "/" + col.getName(), -1);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
new file mode 100644
index 0000000..aef26ec
--- /dev/null
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -0,0 +1,115 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.DateFormat;
+
+import java.io.*;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/6/15.
+ */
+public class TimeStrDictionary extends Dictionary<String> {
+
+ private long maxid = Integer.MAX_VALUE;
+ private int maxLenghOfPositiveLong = 19;
+
+ @Override
+ public int getMinId() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxId() {
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int getSizeOfId() {
+ return 4;
+ }
+
+ @Override
+ public int getSizeOfValue() {
+ return maxLenghOfPositiveLong;
+ }
+
+ @Override
+ protected int getIdFromValueImpl(String value, int roundingFlag) {
+ long millis = DateFormat.stringToMillis(value);
+ long seconds = millis / 1000;
+
+ if (seconds > maxid) {
+ return nullId();
+ } else if (seconds < 0) {
+ throw new IllegalArgumentException("Illegal value: " + value + ", parsed seconds: " + seconds);
+ }
+
+ return (int) seconds;
+ }
+
+ /**
+ *
+ * @param id
+ * @return return like "0000001430812800000"
+ */
+ @Override
+ protected String getValueFromIdImpl(int id) {
+ if (id == nullId())
+ return null;
+
+ long millis = 1000L * id;
+ String s = Long.toString(millis);
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < maxLenghOfPositiveLong - s.length(); ++i) {
+ sb.append('0');
+ }
+ sb.append(s);
+ return sb.toString();
+ }
+
+ @Override
+ final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ try {
+ return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ @Override
+ final protected byte[] getValueBytesFromIdImpl(int id) {
+ String date = getValueFromId(id);
+ byte bytes[];
+ try {
+ bytes = date.getBytes("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ return bytes;
+ }
+
+ @Override
+ final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ byte bytes[] = getValueBytesFromIdImpl(id);
+ System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
+ return bytes.length;
+ }
+
+ @Override
+ public void dump(PrintStream out) {
+ out.println(this.toString());
+ }
+
+ @Override
+ public String toString() {
+ return "TimeStrDictionary supporting from 1970-01-01 00:00:00 to 2038/01/19 03:14:07 (does not support millisecond)";
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
new file mode 100644
index 0000000..e723126
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
@@ -0,0 +1,61 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.DateFormat;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Date;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/6/15.
+ */
+public class TimeStrDictionaryTests {
+ TimeStrDictionary dict;
+
+ @Before
+ public void setup() {
+ dict = new TimeStrDictionary();
+ }
+
+ @Test
+ public void basicTest() {
+ int a = dict.getIdFromValue("1999-01-01");
+ int b = dict.getIdFromValue("1999-01-01 00:00:00");
+ int c = dict.getIdFromValue("1999-01-01 00:00:00.000");
+ int d = dict.getIdFromValue("1999-01-01 00:00:00.022");
+
+ Assert.assertEquals(a, b);
+ Assert.assertEquals(a, c);
+ Assert.assertEquals(a, d);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ encodeDecode("1999-01-12", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("2038-01-09", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("2038-01-08", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("1970-01-01", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("1970-01-02", DateFormat.DEFAULT_DATE_PATTERN);
+
+ encodeDecode("1999-01-12 11:00:01", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("2038-01-09 01:01:02", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("2038-01-19 03:14:07", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("1970-01-01 23:22:11", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("1970-01-02 23:22:11", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ }
+
+ @Test
+ public void testIllegal() {
+ Assert.assertEquals(-1, dict.getIdFromValue("2038-01-19 03:14:08"));
+ }
+
+ public void encodeDecode(String origin, String pattern) {
+ int a = dict.getIdFromValue(origin);
+ String v = dict.getValueFromId(a);
+ long ts = Long.parseLong(v);
+ String back = DateFormat.dateToString(new Date(ts), pattern);
+ Assert.assertEquals(origin, back);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index c57e91f..052b6bb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -71,16 +71,16 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumn().getDatatype())).getLength();
+ lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumnDesc().getDatatype())).getLength();
} else {
if (Array.isEmpty(dictionaryMap)) {
- final DataType dataType = DataType.getInstance(tblColRef.getColumn().getDatatype());
+ final DataType dataType = DataType.getInstance(tblColRef.getColumnDesc().getDatatype());
if (dataType.isNumberFamily()) {
lengths[i] = 16;
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 23;
+ lengths[i] = 19;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 4300140..8dbaed7 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -143,7 +143,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 23;
+ lengths[i] = 19;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index ae563ac..b4a93a1 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -178,7 +178,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
// create join infos
LinkedList<Integer> fkIndice = new LinkedList<Integer>();
for (TblColRef colRef : join.getForeignKeyColumns()) {
- fkIndice.add(colRef.getColumn().getZeroBasedIndex());
+ fkIndice.add(colRef.getColumnDesc().getZeroBasedIndex());
}
this.tableJoins.add(new TableJoin(joinType, fkIndice, lookupTableName, this.findColumnRowKeyRelationships(dim)));
@@ -200,7 +200,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) {
LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();
for (TblColRef colRef : dim.getColumnRefs()) {
- int dimTableIndex = colRef.getColumn().getZeroBasedIndex();
+ int dimTableIndex = colRef.getColumnDesc().getZeroBasedIndex();
int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName());
dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index b5818c4..1d0b16b 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -121,7 +121,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(), lfn.getColumnDesc().getZeroBasedIndex(), "string", signature, "");
List<byte[]> values = new ArrayList<byte[]>();
values.add(new byte[] { 97, 97, 97 });
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index fde41b1..309b6bb 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -18,18 +18,18 @@
package org.apache.kylin.metadata.filter;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
/**
*
* @author xjiang
@@ -89,7 +89,7 @@ public class ColumnTupleFilter extends TupleFilter {
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
- String columnId = columnRef.getColumn().getId();
+ String columnId = columnRef.getColumnDesc().getId();
BytesUtil.writeUTFString(columnId, buffer);
String columnName = columnRef.getName();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 57b50b7..fc7fd2c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -18,17 +18,13 @@
package org.apache.kylin.metadata.filter;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.nio.ByteBuffer;
+import java.util.*;
+
/**
* @author xjiang
*/
@@ -94,6 +90,15 @@ public class CompareTupleFilter extends TupleFilter {
return conditionValues;
}
+ public void updateValues(Collection<?> values, boolean clear) {
+ if (clear) {
+ this.conditionValues.clear();
+ }
+
+ this.conditionValues.addAll(values);
+ this.firstCondValue = this.conditionValues.iterator().next();
+ }
+
public Object getFirstValue() {
return firstCondValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index cc3add2..2cb6e8e 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -18,14 +18,14 @@
package org.apache.kylin.metadata.filter;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
/**
*
* @author xjiang
@@ -78,6 +78,7 @@ public class ConstantTupleFilter extends TupleFilter {
return this.constantValues;
}
+
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public byte[] serialize(IFilterCodeSystem cs) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 76c3911..cbc6ce5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -67,7 +67,7 @@ public class TblColRef {
this.column = column;
}
- public ColumnDesc getColumn() {
+ public ColumnDesc getColumnDesc() {
return column;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 2fa7898..aa2ba30 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -208,7 +208,7 @@ class ProjectL2Cache {
return false;
}
ColumnDesc foundCol = table.findColumnByName(col.getName());
- if (col.getColumn().equals(foundCol) == false) {
+ if (col.getColumnDesc().equals(foundCol) == false) {
logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
return false;
}
@@ -239,7 +239,7 @@ class ProjectL2Cache {
TableCache tableCache = prjCache.tables.get(col.getTable());
prjCache.exposedTables.add(tableCache.tableDesc);
tableCache.exposed = true;
- tableCache.exposedColumns.add(col.getColumn());
+ tableCache.exposedColumns.add(col.getColumnDesc());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index 969259d..695ac8f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -1,12 +1,13 @@
package org.apache.kylin.metadata.realization;
-import com.google.common.base.Function;
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Range;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.common.util.DateFormat;
+
+import com.google.common.base.Function;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
/**
* Created by Hongbin Ma(Binmahone) on 4/14/15.
@@ -59,7 +60,7 @@ public class SQLDigestUtil {
}
private static TupleFilter createFilterForRealtime(TupleFilter originFilter, TblColRef partitionColRef, Range<Long> tsRange) {
- DataType type = partitionColRef.getColumn().getType();
+ DataType type = partitionColRef.getColumnDesc().getType();
String startTimeStr, endTimeStr;
CompareTupleFilter startFilter = null, endFilter = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index f15078a..6f8f0f7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -29,6 +29,7 @@ import net.hydromatic.optiq.runtime.SqlFunctions;
import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
@@ -39,7 +40,10 @@ import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.util.NlsString;
-import java.util.*;
+import java.util.GregorianCalendar;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
/**
* @author xjiang
@@ -188,7 +192,8 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
strValue = ((NlsString) literalValue).getValue();
} else if (literalValue instanceof GregorianCalendar) {
GregorianCalendar g = (GregorianCalendar) literalValue;
- strValue = "" + g.get(Calendar.YEAR) + "-" + normToTwoDigits(g.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(g.get(Calendar.DAY_OF_MONTH));
+ //strValue = "" + g.get(Calendar.YEAR) + "-" + normToTwoDigits(g.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(g.get(Calendar.DAY_OF_MONTH));
+ strValue = Long.toString(g.getTimeInMillis());
} else if (literalValue instanceof SqlFunctions.TimeUnitRange) {
// Extract(x from y) in where clause
strValue = ((SqlFunctions.TimeUnitRange) literalValue).name();
@@ -255,6 +260,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
TupleFilterVisitor visitor = new TupleFilterVisitor(this.columnRowType, context);
context.filter = this.condition.accept(visitor);
+ DateConditionInplaceModifier.modify(context.filter);
context.filterColumns = collectColumns(context.filter);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 6a20bdb..e95da2c 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -95,7 +95,7 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sample.sql";
+ String queryFileName = "src/test/resources/query/sql_orderby/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
index ebb6ce9..6858a67 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
@@ -24,7 +24,7 @@ public class GTFilterScanner implements IGTScanner {
this.oneTuple = new IEvaluatableTuple() {
@Override
public Object getValue(TblColRef col) {
- return next.get(col.getColumn().getZeroBasedIndex());
+ return next.get(col.getColumnDesc().getZeroBasedIndex());
}
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index b33a596..954e464 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -104,7 +104,7 @@ public class GTInfo {
}
public void validateColRef(TblColRef ref) {
- TblColRef expected = colRef(ref.getColumn().getZeroBasedIndex());
+ TblColRef expected = colRef(ref.getColumnDesc().getZeroBasedIndex());
if (expected != ref)
throw new IllegalArgumentException();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
index 449b174..7c6abec 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -193,7 +193,7 @@ public class GTInvertedIndex {
}
private int col(CompareTupleFilter filter) {
- return filter.getColumn().getColumn().getZeroBasedIndex();
+ return filter.getColumn().getColumnDesc().getZeroBasedIndex();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
index fe67a19..d3418f7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
@@ -73,7 +73,7 @@ public class GTScanRangePlanner {
List<GTRecord> hbaseFuzzyKeys = Lists.newArrayList();
for (ColumnRange range : andDimRanges) {
- int col = range.column.getColumn().getZeroBasedIndex();
+ int col = range.column.getColumnDesc().getZeroBasedIndex();
if (info.primaryKey.get(col) == false)
continue;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
index 64e6b90..14af632 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
@@ -91,7 +91,7 @@ public class GTScanRequest {
// filter columns must belong to the table
info.validateColRef(col);
// filter columns must be returned to satisfy upper layer evaluation (calcite)
- columns.set(col.getColumn().getZeroBasedIndex());
+ columns.set(col.getColumnDesc().getZeroBasedIndex());
}
// un-evaluatable filter must be removed
@@ -102,7 +102,7 @@ public class GTScanRequest {
// columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation
if (hasAggregation()) {
for (TblColRef col : unevaluableColumns) {
- aggrGroupBy.set(col.getColumn().getZeroBasedIndex());
+ aggrGroupBy.set(col.getColumnDesc().getZeroBasedIndex());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index 2b77ea0..d4936c4 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -98,7 +98,7 @@ public class GTUtil {
newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
Object firstValue = constValues.iterator().next();
- int col = colMapping == null ? externalCol.getColumn().getZeroBasedIndex() : colMapping.indexOf(externalCol);
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
TupleFilter result;
ByteArray code;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
index 0962ff3..4a6b767 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
@@ -163,7 +163,7 @@ public class CubeTupleConverter {
private int[] initDerivedColIdx() {
int[] idx = new int[deriveInfo.columns.length];
for (int i = 0; i < idx.length; i++) {
- idx[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex();
+ idx[i] = deriveInfo.columns[i].getColumnDesc().getZeroBasedIndex();
}
return idx;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index 36a281f..743769f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -65,11 +65,11 @@ public class DerivedFilterTranslator {
assert hostInfo.type == DeriveType.LOOKUP;
assert hostCols.length == pkCols.length;
- int di = derivedCol.getColumn().getZeroBasedIndex();
+ int di = derivedCol.getColumnDesc().getZeroBasedIndex();
int[] pi = new int[pkCols.length];
int hn = hostCols.length;
for (int i = 0; i < hn; i++) {
- pi[i] = pkCols[i].getColumn().getZeroBasedIndex();
+ pi[i] = pkCols[i].getColumnDesc().getZeroBasedIndex();
}
Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index feb36ca..b3b06a7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -18,8 +18,6 @@
package org.apache.kylin.storage.hbase.coprocessor;
-import java.util.Set;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -27,6 +25,8 @@ import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.util.Set;
+
/**
* @author yangli9
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
new file mode 100644
index 0000000..8569058
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
@@ -0,0 +1,63 @@
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/7/15.
+ *
+ * For historical reasons, date types are encoded with DateStrDcitionary
+ * rather than TimeStrDictionary, so the constant in CompareTupleFilter should
+ * be corrected to "yyyy-MM-dd" format for date types
+ */
+public class DateConditionInplaceModifier {
+
+ public static void modify(TupleFilter filter) {
+
+ if (filter instanceof CompareTupleFilter) {
+ CompareTupleFilter cfilter = (CompareTupleFilter) filter;
+ List<? extends TupleFilter> children = cfilter.getChildren();
+
+ if (children == null || children.size() < 1) {
+ throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
+ }
+
+ if (cfilter.getColumn() == null || (!"date".equals(cfilter.getColumn().getColumnDesc().getTypeName()))) {
+ return;
+ }
+
+ int nonConstantChild = 0;
+ boolean firstUpdate = true;
+ for (int i = 0; i < children.size(); ++i) {
+ if (children.get(i) instanceof ConstantTupleFilter) {
+ ConstantTupleFilter constantAsFilter = (ConstantTupleFilter) children.get(i);
+ Collection<?> values = constantAsFilter.getValues();
+ Collection<Object> newValues = Lists.newArrayList();
+ for (Object x : values) {
+ newValues.add(x == null ? null : DateFormat.formatToDateStr(Long.parseLong((String) x)));
+ }
+ cfilter.updateValues(newValues, firstUpdate);
+ firstUpdate=false;
+ } else {
+ nonConstantChild++;
+ }
+ }
+ if (nonConstantChild != 1) {
+ throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
+ }
+
+ return;
+
+ } else {
+ for (TupleFilter child : filter.getChildren()) {
+ modify(child);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 106e6c5..6a7c14a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -18,11 +18,9 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-import java.io.IOException;
-import java.util.*;
-
-import javax.annotation.Nullable;
-
+import com.google.common.base.Function;
+import com.google.common.collect.*;
+import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang3.SerializationUtils;
@@ -55,14 +53,9 @@ import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
-import com.google.common.collect.Sets;
-import com.google.protobuf.ByteString;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
/**
* Created by Hongbin Ma(Binmahone) on 12/2/14.
@@ -208,7 +201,7 @@ public class EndpointTupleIterator implements ITupleIterator {
functionDesc.setReturnType(iiDefaultHLLC);
functionDesc.setReturnDataType(DataType.getInstance(iiDefaultHLLC));
} else {
- functionDesc.setReturnType(column.getColumn().getType().toString());
+ functionDesc.setReturnType(column.getColumnDesc().getType().toString());
functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
}
functionDesc.getParameter().setColRefs(ImmutableList.of(column));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index e89f8e0..7224cfe 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -18,16 +18,15 @@
package org.apache.kylin.storage.tuple;
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
/**
* @author xjiang
*/
@@ -57,8 +56,7 @@ public class Tuple implements ITuple {
@Override
public ITuple makeCopy() {
Tuple ret = new Tuple(this.info);
- for(int i = 0 ; i < this.values.length; ++i)
- {
+ for (int i = 0; i < this.values.length; ++i) {
ret.values[i] = this.values[i];
}
return ret;
@@ -163,6 +161,8 @@ public class Tuple implements ITuple {
Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
long millis = dateValue.getTime();
return millisToEpicDays(millis);// Optiq expects Integer instead of Long. by honma
+ } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+ return Long.parseLong(strValue);
} else if ("tinyint".equals(dataTypeName)) {
return Byte.valueOf(strValue);
} else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
new file mode 100644
index 0000000..204e8c6
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
@@ -0,0 +1,34 @@
+package org.apache.kylin.storage.filter;
+
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/7/15.
+ */
+public class DateConditionInplaceModifierTest extends FilterBaseTest {
+ @Test
+ public void basicTest() {
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef column = new TblColRef(c1);
+
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ constantFilter = new ConstantTupleFilter("946684800000");
+ compareFilter.addChild(constantFilter);
+
+ DateConditionInplaceModifier.modify(compareFilter);
+ Assert.assertEquals("2000-01-01",compareFilter.getFirstValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index ee810ce..2a258df 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -163,7 +163,7 @@ public class StorageTest extends HBaseMetadataTestCase {
info.setField(col.getName(), col, idx++);
}
- TableDesc sourceTable = groups.get(0).getColumn().getTable();
+ TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
for (FunctionDesc func : aggregations) {
TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
info.setField(col.getName(), col, idx++);
[19/50] [abbrv] incubator-kylin git commit: Merge branch
'origin/streaming-localdict'
Posted by sh...@apache.org.
Merge branch 'origin/streaming-localdict'
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dcf0a21c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dcf0a21c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dcf0a21c
Branch: refs/heads/streaming-localdict
Commit: dcf0a21ccac36b34602f89867f31e67d141adb7b
Parents: 0578951 f9b1442
Author: honma <ho...@ebay.com>
Authored: Wed May 6 10:18:22 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:04:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 9 ++++++---
conf/kylin.properties | 4 +++-
.../test_case_data/sandbox/kylin.properties | 3 +++
.../kylin/job/constant/BatchConstants.java | 2 +-
.../apache/kylin/job/cube/CubingJobBuilder.java | 2 +-
.../kylin/job/hadoop/AbstractHadoopJob.java | 2 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 6 +++---
.../hadoop/cube/FactDistinctColumnsReducer.java | 21 +++++---------------
.../cube/FactDistinctHiveColumnsMapper.java | 19 +++++++-----------
.../kylin/job/streaming/StreamingBootstrap.java | 6 +++++-
10 files changed, 35 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dcf0a21c/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
[09/50] [abbrv] incubator-kylin git commit: KYLIN-749 bug fix
Posted by sh...@apache.org.
KYLIN-749 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/59cb3c56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/59cb3c56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/59cb3c56
Branch: refs/heads/streaming-localdict
Commit: 59cb3c56891a82f64c49bad297386eb2b2a69bdc
Parents: 09aba28
Author: honma <ho...@ebay.com>
Authored: Tue May 5 18:00:15 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 18:00:15 2015 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamingBootstrap.java | 19 ++++++++-----------
.../java/org/apache/kylin/job/DataGenTest.java | 4 +++-
2 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59cb3c56/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 1688fc2..1f7608b 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,19 +34,13 @@
package org.apache.kylin.job.streaming;
-import java.lang.reflect.Constructor;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
import kafka.javaapi.PartitionMetadata;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
@@ -56,8 +50,11 @@ import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
/**
* Created by qianzhou on 3/26/15.
@@ -140,7 +137,7 @@ public class StreamingBootstrap {
streamingOffset = Math.max(streamingOffset, earliestOffset);
logger.info("starting offset is " + streamingOffset);
- IICreateHTableJob.main(new String[] { "-iiname", "nous_ii", "-htablename", "KYLIN_2SKJ8JNOUS" });
+ IICreateHTableJob.main(new String[] { "-iiname", kafkaConfig.getIiName(), "-htablename", iiSegment.getStorageLocationIdentifier() });
KafkaConsumer consumer = new KafkaConsumer(kafkaConfig.getTopic(), partitionId, streamingOffset, kafkaConfig.getBrokers(), kafkaConfig, parallelism);
kafkaConsumers.put(getKey(streaming, partitionId), consumer);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59cb3c56/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index 4e3cf17..7b10a22 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -72,7 +72,7 @@ public class DataGenTest extends LocalFileMetadataTestCase {
int totalCount = 10000;
int counter = 0;
- Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-01-06"), totalCount);
+ Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-02-05"), totalCount);
iterator = SortUtil.extractAndSort(iterator, new Function<String, Comparable>() {
public Comparable apply(String input) {
@@ -80,6 +80,8 @@ public class DataGenTest extends LocalFileMetadataTestCase {
}
});
+ //FileUtils.writeLines(new File("//Users/honma/streaming_table_data"),Lists.newArrayList(iterator));
+
long lastTs = 0;
while (iterator.hasNext()) {
counter++;
[04/50] [abbrv] incubator-kylin git commit: KYLIN-748 some test
cases's column datatype is null
Posted by sh...@apache.org.
KYLIN-748 some test cases's column datatype is null
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0ab01667
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0ab01667
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0ab01667
Branch: refs/heads/streaming-localdict
Commit: 0ab016676366e2bca6a5491496dbfb764c86c2ae
Parents: 5a19aae
Author: honma <ho...@ebay.com>
Authored: Tue May 5 10:09:32 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 10:09:32 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/metadata/model/ColumnDesc.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0ab01667/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index b70ef86..8d7ebfd 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -135,7 +135,12 @@ public class ColumnDesc {
if (id != null)
zeroBasedIndex = Integer.parseInt(id) - 1;
- this.setDatatype(DataType.getInstance(datatype).toString());
+ DataType normalized = DataType.getInstance(datatype);
+ if (normalized == null) {
+ this.setDatatype(null);
+ } else {
+ this.setDatatype(normalized.toString());
+ }
}
public boolean isSameAs(String tableName, String columnName) {
[24/50] [abbrv] incubator-kylin git commit: KYLIN-750 Merge cube
segments from HBase table
Posted by sh...@apache.org.
KYLIN-750 Merge cube segments from HBase table
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/39ada70f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/39ada70f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/39ada70f
Branch: refs/heads/streaming-localdict
Commit: 39ada70f4732203fa599a1cf323f6ddecd73a8ff
Parents: 32ceeee
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu May 7 15:05:29 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu May 7 17:25:56 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/kv/RowValueDecoder.java | 34 ++-
.../apache/kylin/job/cube/CubingJobBuilder.java | 56 ++++-
.../hadoop/cubev2/MergeCuboidFromHBaseJob.java | 123 ++++++++++
.../cubev2/MergeCuboidFromHBaseMapper.java | 246 +++++++++++++++++++
.../kylin/metadata/model/MeasureDesc.java | 2 +-
5 files changed, 440 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39ada70f/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
index ace347a..99c2485 100644
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
+++ b/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
@@ -18,10 +18,6 @@
package org.apache.kylin.cube.kv;
-import java.nio.ByteBuffer;
-import java.util.BitSet;
-import java.util.Collection;
-
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.DoubleWritable;
@@ -33,17 +29,19 @@ import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Collection;
+
/**
- *
* @author xjiang
- *
*/
public class RowValueDecoder implements Cloneable {
private final HBaseColumnDesc hbaseColumn;
private final byte[] hbaseColumnFamily;
private final byte[] hbaseColumnQualifier;
-
+
private final MeasureCodec codec;
private final BitSet projectionIndex;
private final MeasureDesc[] measures;
@@ -60,19 +58,29 @@ public class RowValueDecoder implements Cloneable {
}
public void decode(Result hbaseRow) {
- decode(hbaseRow.getValueAsByteBuffer(hbaseColumnFamily, hbaseColumnQualifier));
+ decode(hbaseRow, true);
+ }
+
+ public void decode(Result hbaseRow, boolean convertToJavaObject) {
+ decode(hbaseRow.getValueAsByteBuffer(hbaseColumnFamily, hbaseColumnQualifier), convertToJavaObject);
}
-
+
public void decode(byte[] bytes) {
- decode(ByteBuffer.wrap(bytes));
+ decode(bytes, true);
+ }
+
+ public void decode(byte[] bytes, boolean convertToJavaObject) {
+ decode(ByteBuffer.wrap(bytes), convertToJavaObject);
}
- private void decode(ByteBuffer buffer) {
+ private void decode(ByteBuffer buffer, boolean convertToJavaObject) {
codec.decode(buffer, values);
- convertToJavaObjects(values, values);
+ if (convertToJavaObject) {
+ convertToJavaObjects(values, values, convertToJavaObject);
+ }
}
- private void convertToJavaObjects(Object[] mapredObjs, Object[] results) {
+ private void convertToJavaObjects(Object[] mapredObjs, Object[] results, boolean convertToJavaObject) {
for (int i = 0; i < mapredObjs.length; i++) {
Object o = mapredObjs[i];
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39ada70f/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index 27613df..e4a0fae 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -32,6 +32,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.hadoop.cube.*;
import org.apache.kylin.job.hadoop.cubev2.InMemCuboidJob;
+import org.apache.kylin.job.hadoop.cubev2.MergeCuboidFromHBaseJob;
import org.apache.kylin.job.hadoop.dict.CreateDictionaryJob;
import org.apache.kylin.job.hadoop.hbase.BulkLoadJob;
import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
@@ -69,8 +70,6 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
if (this.inMemoryCubing()) {
- // bulk load step
- result.addTask(createBulkLoadStep(seg, result.getId()));
result.addTask(createUpdateCubeInfoAfterBuildStep(seg, intermediateHiveTableStepId, baseCuboidStepId, baseCuboidStepId, jobId));
} else {
// convert htable
@@ -103,16 +102,24 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
List<String> mergingSegmentIds = Lists.newArrayList();
List<String> mergingCuboidPaths = Lists.newArrayList();
+ List<String> mergingHTables = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
+ mergingHTables.add(merging.getStorageLocationIdentifier());
if (merging.equals(appendSegment))
mergingCuboidPaths.add(appendRootPath + "*");
else
mergingCuboidPaths.add(getPathToMerge(merging));
}
- // merge cuboid
- addMergeSteps(mergeSegment, mergingSegmentIds, mergingCuboidPaths, mergedRootPath, result);
+ if(this.inMemoryCubing()) {
+ // merge from HTable
+ addMergeFromHBaseSteps(mergeSegment, mergingSegmentIds, mergingHTables, mergedRootPath, result);
+ } else {
+ // merge cuboid
+ addMergeSteps(mergeSegment, mergingSegmentIds, mergingCuboidPaths, mergedRootPath, result);
+ }
+
// convert htable
AbstractExecutable convertCuboidToHfileStep = addHTableSteps(mergeSegment, mergedRootPath, result);
@@ -134,14 +141,20 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
List<String> mergingSegmentIds = Lists.newArrayList();
List<String> mergingCuboidPaths = Lists.newArrayList();
+ List<String> mergingHTables = Lists.newArrayList();
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
mergingCuboidPaths.add(getPathToMerge(merging));
+ mergingHTables.add(merging.getStorageLocationIdentifier());
}
- // merge cuboid
- addMergeSteps(seg, mergingSegmentIds, mergingCuboidPaths, mergedCuboidPath, result);
-
+ if(this.inMemoryCubing()) {
+ // merge from HTable
+ addMergeFromHBaseSteps(seg, mergingSegmentIds, mergingHTables, mergedCuboidPath, result);
+ } else {
+ // merge cuboid
+ addMergeSteps(seg, mergingSegmentIds, mergingCuboidPaths, mergedCuboidPath, result);
+ }
// convert htable
AbstractExecutable convertCuboidToHfileStep = addHTableSteps(seg, mergedCuboidPath, result);
@@ -159,6 +172,15 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
result.addTask(createMergeCuboidDataStep(seg, formattedPath, mergedCuboidPath));
}
+
+ void addMergeFromHBaseSteps(CubeSegment seg, List<String> mergingSegmentIds, List<String> mergingHTables, String mergedCuboidPath, CubingJob result) {
+
+ result.addTask(createMergeDictionaryStep(seg, mergingSegmentIds));
+
+ String formattedTables = StringUtils.join(mergingHTables, ",");
+ result.addTask(createMergeCuboidDataFromHBaseStep(seg, formattedTables, mergedCuboidPath));
+ }
+
Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) {
final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels();
final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length;
@@ -190,6 +212,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
result.addTask(createCreateHTableStep(seg, jobId));
baseCuboidStep = createInMemCubingStep(seg, intermediateHiveTableLocation, intermediateHiveTableName, cuboidOutputTempPath, result.getId());
result.addTask(baseCuboidStep);
+ // bulk load step
+ result.addTask(createBulkLoadStep(seg, result.getId()));
}
return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep);
@@ -480,6 +504,24 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return mergeCuboidDataStep;
}
+
+ private MapReduceExecutable createMergeCuboidDataFromHBaseStep(CubeSegment seg, String inputPath, String outputPath) {
+ MapReduceExecutable mergeCuboidDataStep = new MapReduceExecutable();
+ mergeCuboidDataStep.setName(ExecutableConstants.STEP_NAME_MERGE_CUBOID);
+ StringBuilder cmd = new StringBuilder();
+
+ appendMapReduceParameters(cmd, seg);
+ appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
+ appendExecCmdParameters(cmd, "segmentname", seg.getName());
+ appendExecCmdParameters(cmd, "input", inputPath);
+ appendExecCmdParameters(cmd, "output", outputPath);
+ appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
+
+ mergeCuboidDataStep.setMapReduceParams(cmd.toString());
+ mergeCuboidDataStep.setMapReduceJobClass(MergeCuboidFromHBaseJob.class);
+ return mergeCuboidDataStep;
+ }
+
private UpdateCubeInfoAfterMergeStep createUpdateCubeInfoAfterMergeStep(CubeSegment seg, List<String> mergingSegmentIds, String convertToHFileStepId, String jobId) {
UpdateCubeInfoAfterMergeStep result = new UpdateCubeInfoAfterMergeStep();
result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39ada70f/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
new file mode 100644
index 0000000..07d16f8
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cubev2;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.constant.BatchConstants;
+import org.apache.kylin.job.hadoop.cube.CuboidJob;
+import org.apache.kylin.job.hadoop.cube.CuboidReducer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author shaoshi
+ */
+public class MergeCuboidFromHBaseJob extends CuboidJob {
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_JOB_NAME);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_SEGMENT_NAME);
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ parseOptions(options, args);
+
+ String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+ String segmentName = getOptionValue(OPTION_SEGMENT_NAME).toUpperCase();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+
+
+ Configuration conf = this.getConf();
+ HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
+
+ // start job
+ String jobName = getOptionValue(OPTION_JOB_NAME);
+ System.out.println("Starting: " + jobName);
+ job = Job.getInstance(conf, jobName);
+
+ setJobClasspath(job);
+
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+ FileOutputFormat.setOutputPath(job, output);
+
+
+ List<Scan> scans = new ArrayList<Scan>();
+ for (String htable : StringSplitter.split(getOptionValue(OPTION_INPUT_PATH), ",")) {
+ Scan scan = new Scan();
+ scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs
+ scan.setCacheBlocks(false); // don't set to true for MR jobs
+ scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(htable));
+ scans.add(scan);
+ }
+
+ TableMapReduceUtil.initTableMapperJob(scans, MergeCuboidFromHBaseMapper.class, Text.class,
+ Text.class, job);
+
+ // Reducer - only one
+ job.setReducerClass(CuboidReducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // set job configuration
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+
+ // add metadata to distributed cache
+ attachKylinPropsAndMetadata(cube, job.getConfiguration());
+
+ setReduceTaskNum(job, config, cubeName, 0);
+
+ this.deletePath(job.getConfiguration(), output);
+
+ return waitForCompletion(job);
+ } catch (
+ Exception e
+ )
+
+ {
+ logger.error("error in MergeCuboidFromHBaseJob", e);
+ printUsage(options);
+ throw e;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39ada70f/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
new file mode 100644
index 0000000..cc71351
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cubev2;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.mapreduce.TableSplit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.kv.RowValueDecoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.job.constant.BatchConstants;
+import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.apache.kylin.job.hadoop.cube.KeyValueCreator;
+import org.apache.kylin.metadata.measure.MeasureCodec;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * @author shaoshi
+ */
+public class MergeCuboidFromHBaseMapper extends TableMapper<Text, Text> {
+
+ private KylinConfig config;
+ private String cubeName;
+ private String segmentName;
+ private CubeManager cubeManager;
+ private CubeInstance cube;
+ private CubeDesc cubeDesc;
+ private CubeSegment mergedCubeSegment;
+ private CubeSegment sourceCubeSegment;// Must be unique during a mapper's
+ // life cycle
+
+ private Text outputKey = new Text();
+
+ private byte[] newKeyBuf;
+ private RowKeySplitter rowKeySplitter;
+
+ private HashMap<TblColRef, Boolean> dictsNeedMerging = new HashMap<TblColRef, Boolean>();
+
+ private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+
+ private Text outputValue = new Text();
+ private RowValueDecoder[] rowValueDecoders;
+ private boolean simpleFullCopy = false;
+ private Object[] result;
+ private MeasureCodec codec;
+ private int[][] hbaseColumnToMeasureMapping;
+
+ private Boolean checkNeedMerging(TblColRef col) throws IOException {
+ Boolean ret = dictsNeedMerging.get(col);
+ if (ret != null)
+ return ret;
+ else {
+ ret = cubeDesc.getRowkey().isUseDictionary(col);
+ if (ret) {
+ String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
+ }
+ dictsNeedMerging.put(col, ret);
+ return ret;
+ }
+ }
+
+
+ private CubeSegment findSegmentWithHTable(String htable, CubeInstance cubeInstance) {
+ for (CubeSegment segment : cubeInstance.getSegments()) {
+ String segmentHtable = segment.getStorageLocationIdentifier();
+ if (segmentHtable != null && segmentHtable.equalsIgnoreCase(htable)) {
+ return segment;
+ }
+ }
+
+ throw new IllegalStateException("No merging segment's storage location identifier equals " + htable);
+
+ }
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ HadoopUtil.setCurrentConfiguration(context.getConfiguration());
+
+ cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+ segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME).toUpperCase();
+
+ config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+
+ cubeManager = CubeManager.getInstance(config);
+ cube = cubeManager.getCube(cubeName);
+ cubeDesc = cube.getDescriptor();
+ mergedCubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
+
+ newKeyBuf = new byte[256];// size will auto-grow
+
+ TableSplit currentSplit = (TableSplit) context.getInputSplit();
+ byte[] tableName = currentSplit.getTableName();
+ String htableName = Bytes.toString(tableName);
+ // decide which source segment
+ System.out.println("htable:" + htableName);
+ sourceCubeSegment = findSegmentWithHTable(htableName, cube);
+ System.out.println(sourceCubeSegment);
+
+ this.rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
+
+ List<RowValueDecoder> valueDecoderList = Lists.newArrayList();
+ List<KeyValueCreator> keyValueCreators = Lists.newArrayList();
+ for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
+ valueDecoderList.add(new RowValueDecoder(colDesc));
+ keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
+ }
+ }
+
+ rowValueDecoders = valueDecoderList.toArray(new RowValueDecoder[valueDecoderList.size()]);
+
+ simpleFullCopy = (keyValueCreators.size() == 1 && keyValueCreators.get(0).isFullCopy);
+ result = new Object[cubeDesc.getMeasures().size()];
+ codec = new MeasureCodec(cubeDesc.getMeasures());
+
+ hbaseColumnToMeasureMapping = new int[rowValueDecoders.length][];
+
+ for (int i = 0; i < rowValueDecoders.length; i++) {
+ hbaseColumnToMeasureMapping[i] = new int[rowValueDecoders[i].getMeasures().length];
+ for (int j = 0; j < rowValueDecoders[i].getMeasures().length; j++) {
+ int positionInCubeMeasures = 0;
+ for (MeasureDesc m : cubeDesc.getMeasures()) {
+ if (m.equals(rowValueDecoders[i].getMeasures()[j])) {
+ hbaseColumnToMeasureMapping[i][j] = positionInCubeMeasures;
+ break;
+ }
+ positionInCubeMeasures++;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {
+ long cuboidID = rowKeySplitter.split(key.get(), key.get().length);
+ Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidID);
+
+ SplittedBytes[] splittedByteses = rowKeySplitter.getSplitBuffers();
+ int bufOffset = 0;
+ BytesUtil.writeLong(cuboidID, newKeyBuf, bufOffset, RowConstants.ROWKEY_CUBOIDID_LEN);
+ bufOffset += RowConstants.ROWKEY_CUBOIDID_LEN;
+
+ for (int i = 0; i < cuboid.getColumns().size(); ++i) {
+ TblColRef col = cuboid.getColumns().get(i);
+
+ if (this.checkNeedMerging(col)) {
+ // if dictionary on fact table column, needs rewrite
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
+ Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
+
+ while (sourceDict.getSizeOfValue() > newKeyBuf.length - bufOffset || mergedDict.getSizeOfValue() > newKeyBuf.length - bufOffset) {
+ byte[] oldBuf = newKeyBuf;
+ newKeyBuf = new byte[2 * newKeyBuf.length];
+ System.arraycopy(oldBuf, 0, newKeyBuf, 0, oldBuf.length);
+ }
+
+ int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[i + 1].value, 0, splittedByteses[i + 1].length);
+ int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBuf, bufOffset);
+ int idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
+ BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
+
+ bufOffset += mergedDict.getSizeOfId();
+ } else {
+ // keep as it is
+ while (splittedByteses[i + 1].length > newKeyBuf.length - bufOffset) {
+ byte[] oldBuf = newKeyBuf;
+ newKeyBuf = new byte[2 * newKeyBuf.length];
+ System.arraycopy(oldBuf, 0, newKeyBuf, 0, oldBuf.length);
+ }
+
+ System.arraycopy(splittedByteses[i + 1].value, 0, newKeyBuf, bufOffset, splittedByteses[i + 1].length);
+ bufOffset += splittedByteses[i + 1].length;
+ }
+ }
+ byte[] newKey = Arrays.copyOf(newKeyBuf, bufOffset);
+ outputKey.set(newKey, 0, newKey.length);
+
+ valueBuf.clear();
+ if (simpleFullCopy) {
+ // simple case, should only 1 hbase column and the bytes sequence is same
+ for (Cell cell : value.rawCells()) {
+ valueBuf.put(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ }
+ } else {
+ // complex case, need re-arrange the bytes sequence
+ for (int i = 0; i < rowValueDecoders.length; i++) {
+ rowValueDecoders[i].decode(value, false);
+ Object[] measureValues = rowValueDecoders[i].getValues();
+
+ for (int j = 0; j < measureValues.length; j++) {
+ result[hbaseColumnToMeasureMapping[i][j]] = measureValues[j];
+ }
+ }
+ codec.encode(result, valueBuf);
+ }
+
+ outputValue.set(valueBuf.array(), 0, valueBuf.position());
+ context.write(outputKey, outputValue);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39ada70f/metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index fd42d45..ee4a972 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -80,7 +80,7 @@ public class MeasureDesc {
MeasureDesc that = (MeasureDesc) o;
- if (id != that.id)
+ if (!name.equalsIgnoreCase(that.getName()))
return false;
return true;
[46/50] [abbrv] incubator-kylin git commit: KYLIN-759 refactor
storage layer cache
Posted by sh...@apache.org.
KYLIN-759 refactor storage layer cache
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bc426cc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bc426cc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bc426cc0
Branch: refs/heads/streaming-localdict
Commit: bc426cc008e4424c81c62e7541cd244ac9ad811b
Parents: 6924057
Author: honma <ho...@ebay.com>
Authored: Wed May 13 16:59:45 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 14 20:22:04 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/RangeUtil.java | 14 +-
.../org/apache/kylin/common/util/BasicTest.java | 3 +
.../metadata/tuple/CompoundTupleIterator.java | 25 +--
.../kylin/metadata/tuple/ITupleIterator.java | 14 --
.../metadata/tuple/SimpleTupleIterator.java | 6 -
.../kylin/metadata/tuple/TeeTupleIterator.java | 26 +--
.../metadata/tuple/TeeTupleItrListener.java | 10 +
.../kylin/query/enumerator/OLAPEnumerator.java | 2 +-
.../apache/kylin/storage/IStorageEngine.java | 23 +++
.../kylin/storage/StorageEngineFactory.java | 51 +++--
.../AbstractCacheFledgedStorageEngine.java | 49 +++++
.../cache/CacheFledgedDynamicStorageEngine.java | 148 +++++++++++++++
.../cache/CacheFledgedStaticStorageEngine.java | 80 ++++++++
.../cache/CacheFledgedStorageEngine.java | 186 -------------------
.../kylin/storage/cache/StorageLayerCache.java | 7 -
.../kylin/storage/cache/StreamSQLResult.java | 44 +++--
.../kylin/storage/cube/CubeStorageEngine.java | 22 ++-
.../cube/SerializedCubeTupleIterator.java | 9 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 19 +-
.../kylin/storage/hbase/CubeStorageEngine.java | 47 +++--
.../hbase/InvertedIndexStorageEngine.java | 19 +-
.../hbase/SerializedHBaseTupleIterator.java | 4 -
.../endpoint/EndpointTupleIterator.java | 21 ++-
.../endpoint/TsConditionExtractor.java | 9 +-
.../storage/hybrid/HybridStorageEngine.java | 38 ++--
.../apache/kylin/storage/test/StorageTest.java | 15 +-
26 files changed, 512 insertions(+), 379 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
index 33d6770..05e4d35 100644
--- a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -1,11 +1,11 @@
package org.apache.kylin.common.util;
+import com.google.common.collect.*;
+
import java.util.Collections;
import java.util.List;
import java.util.NavigableMap;
-import com.google.common.collect.*;
-
/**
*/
public class RangeUtil {
@@ -44,6 +44,16 @@ public class RangeUtil {
return range.upperBoundType() == BoundType.CLOSED;
}
+ public static <C extends Comparable<?>> Range<C> merge(Range<C> a, Range<C> b) {
+ if (a == null && b == null) {
+ return null;
+ } else if (a == null || b == null) {
+ return a == null ? b : a;
+ } else {
+ return a.span(b);
+ }
+ }
+
/**
* remove from self the elements that exist in other
* @return
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 2f4943d..e0226f8 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -62,6 +62,9 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
+ Number xx =new Long(0L);
+ System.out.println(xx.getClass().getName());
+
System.out.println(time(1367798400000L));
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1000L * Integer.MAX_VALUE));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
index b87c633..f5d8dd6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
@@ -1,14 +1,13 @@
package org.apache.kylin.metadata.tuple;
-import java.util.Iterator;
-import java.util.List;
-
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
-import com.google.common.collect.Range;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Iterator;
+import java.util.List;
+
/**
*/
public class CompoundTupleIterator implements ITupleIterator {
@@ -30,24 +29,6 @@ public class CompoundTupleIterator implements ITupleIterator {
}
@Override
- public Range<Long> getCacheExcludedPeriod() {
- Range<Long> timeSpan = null;
- for (ITupleIterator itt : this.backends) {
- Range<Long> excluded = itt.getCacheExcludedPeriod();
- if (excluded != null && !excluded.isEmpty()) {
- if (timeSpan == null) {
- //first one
- timeSpan = excluded;
- } else {
- logger.warn("There are two excluded period, use a span to replace them: " + timeSpan + " and " + excluded );
- timeSpan = timeSpan.span(excluded);
- }
- }
- }
- return timeSpan;
- }
-
- @Override
public boolean hasNext() {
return this.compoundIterator.hasNext();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
index 9e92eb9..6bab137 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
@@ -20,8 +20,6 @@ package org.apache.kylin.metadata.tuple;
import java.util.Iterator;
-import com.google.common.collect.Range;
-
/**
* @author xjiang
*
@@ -46,22 +44,10 @@ public interface ITupleIterator extends Iterator<ITuple> {
@Override
public void close() {
}
-
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return null;
- }
};
void close();
- /**
- * tells storage layer cache what time period of data should not be cached.
- * for static storage like cube, it will return null
- * for dynamic storage like ii, it will for example exclude the last two minutes for possible data latency
- * @return
- */
- Range<Long> getCacheExcludedPeriod();
/**
* if hasNext() returns false because there's no more data, return true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
index ebdcb5d..92006e1 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
@@ -2,8 +2,6 @@ package org.apache.kylin.metadata.tuple;
import java.util.Iterator;
-import com.google.common.collect.Range;
-
/**
*
*/
@@ -34,8 +32,4 @@ public class SimpleTupleIterator implements ITupleIterator {
public void close() {
}
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
index 556fca6..0618b45 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
@@ -1,10 +1,8 @@
package org.apache.kylin.metadata.tuple;
-import java.util.List;
-
-import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
+
+import java.util.List;
/**
*
@@ -13,29 +11,21 @@ import com.google.common.collect.Range;
*/
public class TeeTupleIterator implements ITupleIterator {
- private Function<List<ITuple>, Void> actionOnSeeingWholeData;
private ITupleIterator underlying;
private List<ITuple> duplicatedData;
+ private List<TeeTupleItrListener> listeners = Lists.newArrayList();
public TeeTupleIterator(ITupleIterator underlying) {
this.underlying = underlying;
this.duplicatedData = Lists.newArrayList();
}
- public void setActionOnSeeingWholeData(Function<List<ITuple>, Void> actionOnSeeingWholeData) {
- this.actionOnSeeingWholeData = actionOnSeeingWholeData;
- }
-
@Override
public void close() {
this.underlying.close();
- //if(this.underlying.isDrained)
- actionOnSeeingWholeData.apply(duplicatedData);
- }
-
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return this.underlying.getCacheExcludedPeriod();
+ for (TeeTupleItrListener listener : this.listeners) {
+ listener.notify(this.duplicatedData);
+ }
}
@Override
@@ -54,4 +44,8 @@ public class TeeTupleIterator implements ITupleIterator {
public void remove() {
this.underlying.remove();
}
+
+ public void addCloseListener(TeeTupleItrListener listener) {
+ this.listeners.add(listener);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
new file mode 100644
index 0000000..50b94bd
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleItrListener.java
@@ -0,0 +1,10 @@
+package org.apache.kylin.metadata.tuple;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/13/15.
+ */
+public interface TeeTupleItrListener {
+ void notify(List<ITuple> duplicated);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index a5019ab..63f06f1 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -112,7 +112,7 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
olapContext.resetSQLDigest();
// query storage engine
- IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization, true);
+ IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization);
ITupleIterator iterator = storageEngine.search(olapContext.storageContext, olapContext.getSQLDigest(), olapContext.returnTupleInfo);
if (logger.isDebugEnabled()) {
logger.debug("return TupleIterator...");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
index 9a88676..d6b777a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
@@ -22,6 +22,8 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.tuple.TupleInfo;
+import com.google.common.collect.Range;
+
/**
*
* @author xjiang
@@ -31,4 +33,25 @@ public interface IStorageEngine {
ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo);
+ /**
+ *
+ * being dynamic => getVolatilePeriod() return not null
+ * being dynamic => partition column of its realization not null
+ *
+ * @return true for static storage like cubes
+ * false for dynamic storage like II
+ */
+ boolean isDynamic();
+
+ /**
+ * volatile period is the period of time in which the returned data is not stable
+ * e.g. inverted index's last several minutes' data is dynamic as time goes by.
+ * data in this period cannot be cached
+ *
+ * This method should not be called before ITupleIterator.close() is called
+ *
+ * @return null if the underlying storage guarantees the data is static
+ */
+ Range<Long> getVolatilePeriod();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
index 4d150ad..29a53fd 100644
--- a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
+++ b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
@@ -18,37 +18,66 @@
package org.apache.kylin.storage;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.storage.cache.CacheFledgedStorageEngine;
+import org.apache.kylin.storage.cache.CacheFledgedDynamicStorageEngine;
+import org.apache.kylin.storage.cache.CacheFledgedStaticStorageEngine;
import org.apache.kylin.storage.hbase.CubeStorageEngine;
import org.apache.kylin.storage.hbase.InvertedIndexStorageEngine;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.apache.kylin.storage.hybrid.HybridStorageEngine;
+import com.google.common.base.Preconditions;
+
/**
* @author xjiang
*/
public class StorageEngineFactory {
+ private static boolean allowStorageLayerCache = true;
- public static IStorageEngine getStorageEngine(IRealization realization, boolean allowCache) {
- if (realization.getType() == RealizationType.CUBE) {
- allowCache = false;
- }
-
- if (allowCache) {
- return new CacheFledgedStorageEngine(realization);
- }
+ public static IStorageEngine getStorageEngine(IRealization realization) {
if (realization.getType() == RealizationType.INVERTED_INDEX) {
- return new InvertedIndexStorageEngine((IIInstance) realization);
+ IStorageEngine ret = new InvertedIndexStorageEngine((IIInstance) realization);
+ if (allowStorageLayerCache) {
+ return wrapWithCache(ret, realization);
+ } else {
+ return ret;
+ }
} else if (realization.getType() == RealizationType.CUBE) {
- return new CubeStorageEngine((CubeInstance) realization);
+ IStorageEngine ret = new CubeStorageEngine((CubeInstance) realization);
+ if (allowStorageLayerCache) {
+ return wrapWithCache(ret, realization);
+ } else {
+ return ret;
+ }
} else {
return new HybridStorageEngine((HybridInstance) realization);
}
}
+ private static IStorageEngine wrapWithCache(IStorageEngine underlyingStorageEngine, IRealization realization) {
+ if (underlyingStorageEngine.isDynamic()) {
+ return new CacheFledgedDynamicStorageEngine(underlyingStorageEngine, getPartitionCol(realization));
+ } else {
+ return new CacheFledgedStaticStorageEngine(underlyingStorageEngine);
+ }
+ }
+
+ private static TblColRef getPartitionCol(IRealization realization) {
+ String modelName = realization.getModelName();
+ DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
+ PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
+ Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
+ TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
+ Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null");
+ return partitionColRef;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
new file mode 100644
index 0000000..d8e2fc0
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
@@ -0,0 +1,49 @@
+package org.apache.kylin.storage.cache;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+
+import org.apache.kylin.metadata.realization.StreamSQLDigest;
+import org.apache.kylin.storage.IStorageEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/13/15.
+ */
+public abstract class AbstractCacheFledgedStorageEngine {
+ private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFledgedStorageEngine.class);
+ protected static CacheManager cacheManager = CacheManager.create();
+
+ protected final IStorageEngine underlyingStorage;
+ protected StreamSQLDigest streamSQLDigest;
+ protected boolean queryCacheExists;
+
+ public AbstractCacheFledgedStorageEngine(IStorageEngine underlyingStorage) {
+ this.underlyingStorage = underlyingStorage;
+ this.queryCacheExists = false;
+ this.makeCacheIfNecessary(underlyingStorage.getClass().getName());
+ }
+
+ private void makeCacheIfNecessary(String storageClassName) {
+ if (cacheManager.getCache(storageClassName) == null) {
+ logger.info("Cache for {} initting:", storageClassName);
+ // TODO: L4J [2015-04-20 10:44:03,817][WARN][net.sf.ehcache.pool.sizeof.ObjectGraphWalker] - The configured limit of 1,000 object references was reached while attempting to calculate the size of the object graph. Severe performance degradation could occur if the sizing operation continues. This can be avoided by setting the CacheManger or Cache <sizeOfPolicy> elements maxDepthExceededBehavior to "abort" or adding stop points with @IgnoreSizeOf annotations. If performance degradation is NOT an issue at the configured limit, raise the limit value using the CacheManager or Cache <sizeOfPolicy
+ //Create a Cache specifying its configuration.
+ Cache storageCache = new Cache(new CacheConfiguration(storageClassName, 0).//
+ memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).//
+ eternal(false).//
+ timeToIdleSeconds(86400).//
+ diskExpiryThreadIntervalSeconds(0).//
+ maxBytesLocalHeap(256, MemoryUnit.MEGABYTES).//
+ persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
+ //TODO: deal with failed queries, and only cache too long query
+
+ cacheManager.addCache(storageCache);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
new file mode 100644
index 0000000..0d7ab18
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
@@ -0,0 +1,148 @@
+package org.apache.kylin.storage.cache;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.Element;
+import org.apache.kylin.common.util.RangeUtil;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.SQLDigestUtil;
+import org.apache.kylin.metadata.realization.StreamSQLDigest;
+import org.apache.kylin.metadata.tuple.*;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/11/15.
+ */
+public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
+ private static final Logger logger = LoggerFactory.getLogger(CacheFledgedDynamicStorageEngine.class);
+
+ private final TblColRef partitionColRef;
+
+ private Range<Long> ts;
+
+ public CacheFledgedDynamicStorageEngine(IStorageEngine underlyingStorage, TblColRef partitionColRef) {
+ super(underlyingStorage);
+ this.partitionColRef = partitionColRef;
+
+ Preconditions.checkArgument(this.partitionColRef != null, "For dynamic columns like " + //
+ this.underlyingStorage.getClass().getName() + ", partition column must be provided");
+ }
+
+ @Override
+ public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
+ //if isDynamicStorage && query involves filter on partition column, the cache requires updating after query done.
+ boolean needUpdateCache = sqlDigest.groupbyColumns.contains(partitionColRef);
+
+ streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
+ StreamSQLResult cachedResult = null;
+ Cache cache = cacheManager.getCache(this.underlyingStorage.getClass().getName());
+ Element element = cache.get(streamSQLDigest);
+ if (element != null) {
+ this.queryCacheExists = true;
+ cachedResult = (StreamSQLResult) element.getObjectValue();
+ }
+
+ ts = TsConditionExtractor.extractTsCondition(partitionColRef, sqlDigest.filter);
+ if (ts == null || ts.isEmpty()) {
+ logger.info("ts range in the query conflicts,return empty directly");
+ return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+ }
+
+ ITupleIterator ret = null;
+ if (cachedResult != null) {
+ Range<Long> reusePeriod = cachedResult.getReusableResults(ts);
+
+ logger.info("existing cache : " + cachedResult);
+ logger.info("ts Range in query: " + RangeUtil.formatTsRange(ts));
+ logger.info("potential reusable range : " + RangeUtil.formatTsRange(reusePeriod));
+
+ if (reusePeriod != null) {
+ List<Range<Long>> remainings = RangeUtil.remove(ts, reusePeriod);
+ if (remainings.size() == 1) {//if using cache causes two underlyingStorage searches, we'd rather not use the cache
+
+ SimpleTupleIterator reusedTuples = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
+ List<ITupleIterator> iTupleIteratorList = Lists.newArrayList();
+ iTupleIteratorList.add(reusedTuples);
+
+ for (Range<Long> remaining : remainings) {
+ logger.info("Appending ts " + RangeUtil.formatTsRange(remaining) + " as additional filter");
+
+ ITupleIterator freshTuples = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, remaining, new Function<Void, ITupleIterator>() {
+ @Override
+ public ITupleIterator apply(Void input) {
+ return underlyingStorage.search(context, sqlDigest, returnTupleInfo);
+ }
+ });
+ iTupleIteratorList.add(freshTuples);
+ }
+
+ ret = new CompoundTupleIterator(iTupleIteratorList);
+ } else if (remainings.size() == 0) {
+ needUpdateCache = false;
+ ret = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
+ } else {
+ //if using cache causes two underlyingStorage searches, we'd rather not use the cache
+ }
+ }
+ } else {
+ logger.info("no cache entry for this query");
+ }
+
+ if (ret == null) {
+ logger.info("decision: not using cache");
+ ret = underlyingStorage.search(context, sqlDigest, returnTupleInfo);
+ } else {
+ logger.info("decision: use cache");
+ }
+
+ if (needUpdateCache || !queryCacheExists) {
+ //use another nested ITupleIterator to deal with cache
+ final TeeTupleIterator tee = new TeeTupleIterator(ret);
+ tee.addCloseListener(this);
+ return tee;
+ } else {
+ return ret;
+ }
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return underlyingStorage.getVolatilePeriod();
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return true;
+ }
+
+ @Override
+ public void notify(List<ITuple> duplicated) {
+ Range<Long> cacheExclude = this.underlyingStorage.getVolatilePeriod();
+ if (cacheExclude != null) {
+ List<Range<Long>> cachablePeriods = RangeUtil.remove(ts, cacheExclude);
+ if (cachablePeriods.size() == 1) {
+ if (!ts.equals(cachablePeriods.get(0))) {
+ logger.info("With respect to each shard's build status, the cacheable tsRange shrinks from " + RangeUtil.formatTsRange(ts) + " to " + RangeUtil.formatTsRange(cachablePeriods.get(0)));
+ }
+ ts = cachablePeriods.get(0);
+ } else {
+ //give up updating the cache, in avoid to make cache complicated
+ }
+ }
+
+ StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, ts, partitionColRef);
+ cacheManager.getCache(this.underlyingStorage.getClass().getName()).put(new Element(streamSQLDigest, newCacheEntry));
+ logger.info("cache after the query: " + newCacheEntry);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
new file mode 100644
index 0000000..a4aceed
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
@@ -0,0 +1,80 @@
+package org.apache.kylin.storage.cache;
+
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.Element;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.StreamSQLDigest;
+import org.apache.kylin.metadata.tuple.*;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/11/15.
+ */
+public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
+ private static final Logger logger = LoggerFactory.getLogger(CacheFledgedStaticStorageEngine.class);
+
+ public CacheFledgedStaticStorageEngine(IStorageEngine underlyingStorage) {
+ super(underlyingStorage);
+ }
+
+ @Override
+ public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
+
+ streamSQLDigest = new StreamSQLDigest(sqlDigest, null);
+ StreamSQLResult cachedResult = null;
+ Cache cache = cacheManager.getCache(this.underlyingStorage.getClass().getName());
+ Element element = cache.get(streamSQLDigest);
+ if (element != null) {
+ this.queryCacheExists = true;
+ cachedResult = (StreamSQLResult) element.getObjectValue();
+ }
+
+ ITupleIterator ret = null;
+ if (cachedResult != null) {
+ ret = new SimpleTupleIterator(cachedResult.reuse(Ranges.<Long> all()));
+ } else {
+ logger.info("no cache entry for this query");
+ }
+
+ if (ret == null) {
+ logger.info("decision: not using cache");
+ ret = underlyingStorage.search(context, sqlDigest, returnTupleInfo);
+ } else {
+ logger.info("decision: use cache");
+ }
+
+ if (queryCacheExists) {
+ //use another nested ITupleIterator to deal with cache
+ final TeeTupleIterator tee = new TeeTupleIterator(ret);
+ tee.addCloseListener(this);
+ return tee;
+ } else {
+ return ret;
+ }
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return underlyingStorage.getVolatilePeriod();
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
+ @Override
+ public void notify(List<ITuple> duplicated) {
+ StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, Ranges.<Long> all(), null);
+ cacheManager.getCache(this.underlyingStorage.getClass().getName()).put(new Element(streamSQLDigest, newCacheEntry));
+ logger.info("cache after the query: " + newCacheEntry);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
deleted file mode 100644
index 068619e..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
+++ /dev/null
@@ -1,186 +0,0 @@
-package org.apache.kylin.storage.cache;
-
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.MemoryUnit;
-import net.sf.ehcache.config.PersistenceConfiguration;
-import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.RangeUtil;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.realization.SQLDigestUtil;
-import org.apache.kylin.metadata.realization.StreamSQLDigest;
-import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.metadata.tuple.TeeTupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-
-/**
- */
-public class CacheFledgedStorageEngine implements IStorageEngine {
-
- private static final Logger logger = LoggerFactory.getLogger(CacheFledgedStorageEngine.class);
-
- public static final String STORAGE_LAYER_TUPLE_CACHE = "STORAGE_LAYER_TUPLE_CACHE";
- //TODO: deal with failed queries
-
- static CacheManager cacheManager;
-
- static {
- // TODO: L4J [2015-04-20 10:44:03,817][WARN][net.sf.ehcache.pool.sizeof.ObjectGraphWalker] - The configured limit of 1,000 object references was reached while attempting to calculate the size of the object graph. Severe performance degradation could occur if the sizing operation continues. This can be avoided by setting the CacheManger or Cache <sizeOfPolicy> elements maxDepthExceededBehavior to "abort" or adding stop points with @IgnoreSizeOf annotations. If performance degradation is NOT an issue at the configured limit, raise the limit value using the CacheManager or Cache <sizeOfPolicy
- cacheManager = CacheManager.create();
-
- //Create a Cache specifying its configuration.
- Cache successCache = new Cache(new CacheConfiguration(STORAGE_LAYER_TUPLE_CACHE, 0).//
- memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).//
- eternal(false).//
- timeToIdleSeconds(86400).//
- diskExpiryThreadIntervalSeconds(0).//
- maxBytesLocalHeap(1, MemoryUnit.GIGABYTES).//
- persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
-
- cacheManager.addCache(successCache);
- }
-
- private TblColRef partitionColRef;
- private IRealization realization;
-
- public CacheFledgedStorageEngine(IRealization realization) {
- this.realization = realization;
-
- Preconditions.checkArgument(realization.getType() != RealizationType.CUBE, "Cube realization does not need dynamic cache!");
- String modelName = realization.getModelName();
- DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
- PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
- Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
- assert partitionDesc != null;
- partitionColRef = partitionDesc.getPartitionDateColumnRef();
- Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null");
- }
-
- @Override
- public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
-
- //enable storage layer cache iff ts column is contained in filter
- boolean needUpdateCache = sqlDigest.groupbyColumns.contains(partitionColRef);
-
- final StreamSQLDigest streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
- StreamSQLResult cachedResult = null;
- Cache cache = cacheManager.getCache(STORAGE_LAYER_TUPLE_CACHE);
- Element element = cache.get(streamSQLDigest);
- if (element != null) {
- cachedResult = (StreamSQLResult) element.getObjectValue();
- }
-
- Range<Long> ts = TsConditionExtractor.extractTsCondition(partitionColRef, sqlDigest.filter);
- if (ts == null || ts.isEmpty()) {
- logger.info("ts range in the query conflicts,return empty directly");
- return ITupleIterator.EMPTY_TUPLE_ITERATOR;
- }
-
- ITupleIterator ret = null;
- if (cachedResult != null) {
- logger.debug("existing cache : " + cachedResult);
- Range<Long> reusePeriod = cachedResult.getReusableResults(ts);
-
- logger.info("ts Range in query: " + RangeUtil.formatTsRange(ts));
- logger.info("potential reusable range : " + RangeUtil.formatTsRange(reusePeriod));
-
- if (reusePeriod != null) {
-
- List<Range<Long>> remainings = RangeUtil.remove(ts, reusePeriod);
- if (remainings.size() == 1) {
-
- SimpleTupleIterator reusedTuples = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
- Range<Long> remaining = remainings.get(0);
- logger.info("Appending ts " + RangeUtil.formatTsRange(remaining) + " as additional filter");
- ITupleIterator freshTuples = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, remaining, new Function<Void, ITupleIterator>() {
- @Override
- public ITupleIterator apply(Void input) {
- return StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest, returnTupleInfo);
- }
- });
-
- ret = new CompoundTupleIterator(Lists.newArrayList(reusedTuples, freshTuples));
- } else if (remainings.size() == 0) {
- needUpdateCache = false;
- ret = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
- }
- //if remaining size > 1, we skip using cache , i.e, ret will == null
- }
- } else {
- logger.info("no cache entry for this query");
- }
-
- if (ret == null) {
- logger.info("decision: not using cache");
- //cache cannot reuse case:
- ret = StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest, returnTupleInfo);
- } else {
- logger.info("decision: use cache");
- }
-
- if (needUpdateCache) {
- //the tsRange in cache should reflect data aliveness
- final Range<Long> finalTs = ts;
-
- //use another nested ITupleIterator to deal with cache
- final TeeTupleIterator tee = new TeeTupleIterator(ret);
- tee.setActionOnSeeingWholeData(new Function<List<ITuple>, Void>() {
- @Nullable
- @Override
- public Void apply(List<ITuple> input) {
- Range<Long> tsRange = finalTs;
- Range<Long> cacheExclude = tee.getCacheExcludedPeriod();
- if (cacheExclude != null) {
- List<Range<Long>> cachablePeriods = RangeUtil.remove(tsRange, cacheExclude);
- if (cachablePeriods.size() == 1) {
- if (!tsRange.equals(cachablePeriods.get(0))) {
- logger.info("With respect to each shard's build status, the cacheable tsRange shrinks from " + RangeUtil.formatTsRange(tsRange) + " to " + RangeUtil.formatTsRange(cachablePeriods.get(0)));
- }
- tsRange = cachablePeriods.get(0);
- } else {
- //give up updating the cache, in avoid to make cache complicated
- return null;
- }
- }
-
- StreamSQLResult newCacheEntry = new StreamSQLResult(input, tsRange, partitionColRef);
- cacheManager.getCache(STORAGE_LAYER_TUPLE_CACHE).put(new Element(streamSQLDigest, newCacheEntry));
- logger.debug("cache after the query: " + newCacheEntry);
- return null;
- }
- });
-
- return tee;
- } else {
- return ret;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java b/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
deleted file mode 100644
index 9b4caa5..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.kylin.storage.cache;
-
-/**
- */
-public interface StorageLayerCache {
- boolean isCacheEnabled();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 5e58822..e895d8e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -1,22 +1,19 @@
package org.apache.kylin.storage.cache;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NavigableMap;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import org.apache.kylin.common.util.RangeUtil;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
+import org.apache.kylin.common.util.RangeUtil;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.storage.tuple.Tuple;
+
+import javax.annotation.Nullable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableMap;
/**
*/
@@ -27,20 +24,22 @@ public class StreamSQLResult {
public StreamSQLResult(List<ITuple> rows, Range<Long> timeCovered, TblColRef partitionCol) {
- Preconditions.checkArgument(timeCovered.hasUpperBound(),"StreamSQLResult requires timeCovered having a upperBound");
-
sortedRows = Maps.newTreeMap();
for (ITuple row : rows) {
- long t = Tuple.getTs(row,partitionCol);
+ if (partitionCol != null) {
+ long t = Tuple.getTs(row, partitionCol);
- //will only cache rows that are within the time range
- if (timeCovered.contains(t)) {
- if (!this.sortedRows.containsKey(t)) {
- this.sortedRows.put(t, Lists.newArrayList(row));
- } else {
- this.sortedRows.get(t).add(row);
+ //will only cache rows that are within the time range
+ if (timeCovered.contains(t)) {
+ if (!this.sortedRows.containsKey(t)) {
+ this.sortedRows.put(t, Lists.newArrayList(row));
+ } else {
+ this.sortedRows.get(t).add(row);
+ }
}
+ } else {
+ this.sortedRows.get(0L).add(row);
}
}
this.timeCovered = timeCovered;
@@ -76,8 +75,7 @@ public class StreamSQLResult {
return sortedRows.size() + " tuples cached for period " + RangeUtil.formatTsRange(timeCovered);
}
- public long getEndTime()
- {
+ public long getEndTime() {
return this.timeCovered.upperEndpoint();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
index 1633f25..a353f49 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
@@ -1,11 +1,8 @@
package org.apache.kylin.storage.cube;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -31,8 +28,7 @@ import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import java.util.*;
public class CubeStorageEngine implements IStorageEngine {
@@ -97,6 +93,16 @@ public class CubeStorageEngine implements IStorageEngine {
return new SerializedCubeTupleIterator(scanners);
}
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return null;
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
for (FunctionDesc func : sqlDigest.aggregations) {
if (!func.isDimensionAsMetric()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/cube/SerializedCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/SerializedCubeTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/cube/SerializedCubeTupleIterator.java
index e69a1c7..7b9286c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/SerializedCubeTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/SerializedCubeTupleIterator.java
@@ -1,11 +1,10 @@
package org.apache.kylin.storage.cube;
-import java.util.List;
-
-import com.google.common.collect.Range;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import java.util.List;
+
public class SerializedCubeTupleIterator implements ITupleIterator {
public SerializedCubeTupleIterator(List<CubeScanner> scanners) {
@@ -35,9 +34,5 @@ public class SerializedCubeTupleIterator implements ITupleIterator {
}
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index a9b4f92..a8027c0 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -18,23 +18,15 @@
package org.apache.kylin.storage.hbase;
-import java.text.MessageFormat;
-import java.util.*;
-
import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.common.persistence.StorageException;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowValueDecoder;
@@ -49,7 +41,8 @@ import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Range;
+import java.text.MessageFormat;
+import java.util.*;
/**
* @author xjiang
@@ -106,10 +99,6 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
}
}
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return null;
- }
@Override
public boolean hasNext() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index ab75a3c..699cc41 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -18,44 +18,43 @@
package org.apache.kylin.storage.hbase;
-import java.util.*;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.storage.StorageContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
-
+import org.apache.hadoop.hbase.client.HConnection;
import org.apache.kylin.common.persistence.HBaseConnection;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowValueDecoder;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
/**
* @author xjiang, yangli9
@@ -133,6 +132,16 @@ public class CubeStorageEngine implements IStorageEngine {
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
}
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return null;
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
private void buildDimensionsAndMetrics(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, SQLDigest sqlDigest) {
for (FunctionDesc func : sqlDigest.aggregations) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
index 9259db6..c94222e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
@@ -18,8 +18,7 @@
package org.apache.kylin.storage.hbase;
-import java.util.ArrayList;
-
+import com.google.common.collect.Range;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
@@ -33,6 +32,8 @@ import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+
/**
* @author yangli9
*/
@@ -41,6 +42,7 @@ public class InvertedIndexStorageEngine implements IStorageEngine {
private static Logger logger = LoggerFactory.getLogger(InvertedIndexStorageEngine.class);
private IISegment seg;
+ private EndpointTupleIterator dataIterator;
public InvertedIndexStorageEngine(IIInstance ii) {
this.seg = ii.getFirstSegment();
@@ -54,10 +56,21 @@ public class InvertedIndexStorageEngine implements IStorageEngine {
@SuppressWarnings("deprecation")
HConnection conn = HBaseConnection.get(context.getConnUrl());
try {
- return new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn, returnTupleInfo);
+ dataIterator = new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn, returnTupleInfo);
+ return dataIterator;
} catch (Throwable e) {
logger.error("Error when connecting to II htable " + tableName, e);
throw new IllegalStateException("Error when connecting to II htable " + tableName, e);
}
}
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return dataIterator.getCacheExcludedPeriod();
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
index 3bb8b80..9cc9eb2 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
@@ -149,8 +149,4 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
}
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index d9ede57..9cb5f36 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -19,10 +19,10 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.*;
import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -82,7 +82,7 @@ public class EndpointTupleIterator implements ITupleIterator {
private HTableInterface table = null;
private TblColRef partitionCol;
- private long lastDataTime;
+ private long lastDataTime = -1;
private int rowsInAllMetric = 0;
public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn, TupleInfo returnTupleInfo) throws Throwable {
@@ -139,8 +139,7 @@ public class EndpointTupleIterator implements ITupleIterator {
//decompress
Collection<IIProtos.IIResponseInternal> shardResults = new ArrayList<>();
- for(IIProtos.IIResponse input : compressedShardResults)
- {
+ for (IIProtos.IIResponse input : compressedShardResults) {
byte[] compressed = input.getBlob().toByteArray();
try {
byte[] decompressed = CompressionUtils.decompress(compressed);
@@ -251,8 +250,14 @@ public class EndpointTupleIterator implements ITupleIterator {
logger.info("Closed after " + rowsInAllMetric + " rows are fetched");
}
- @Override
+ /**
+ * tells storage layer cache what time period of data should not be cached.
+ * for static storage like cube, it will return null
+ * for dynamic storage like ii, it will for example exclude the last two minutes for possible data latency
+ * @return
+ */
public Range<Long> getCacheExcludedPeriod() {
+ Preconditions.checkArgument(lastDataTime != -1, "lastDataTime is not set yet");
return Ranges.greaterThan(lastDataTime);
}
@@ -333,7 +338,7 @@ public class EndpointTupleIterator implements ITupleIterator {
}
index++;
-
+
return tupleConverter.makeTuple(this.tableRecord, this.measureValues, this.tuple);
}
@@ -346,9 +351,5 @@ public class EndpointTupleIterator implements ITupleIterator {
public void close() {
}
- @Override
- public Range<Long> getCacheExcludedPeriod() {
- throw new NotImplementedException();
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
index 18155c6..27a919c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
@@ -1,13 +1,13 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-import com.google.common.collect.Ranges;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.common.util.DateFormat;
import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
/**
*/
@@ -23,6 +23,9 @@ public class TsConditionExtractor {
}
private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
+ if (filter == null) {
+ return Ranges.all();
+ }
if (filter instanceof LogicalTupleFilter) {
if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
@@ -41,6 +44,8 @@ public class TsConditionExtractor {
}
return ret.isEmpty() ? null : ret;
} else {
+ //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
+ //we will use Ranges.all() rather than two ranges to represent them
return Ranges.all();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 87ff26b..f9cea05 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,12 +1,14 @@
package org.apache.kylin.storage.hybrid;
-import javax.annotation.Nullable;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.RangeUtil;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.SQLDigestUtil;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
@@ -16,25 +18,27 @@ import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageEngineFactory;
import org.apache.kylin.storage.tuple.TupleInfo;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Ranges;
+import javax.annotation.Nullable;
/**
*/
public class HybridStorageEngine implements IStorageEngine {
private HybridInstance hybridInstance;
+ private IStorageEngine historicalStorageEngine;
+ private IStorageEngine realtimeStorageEngine;
public HybridStorageEngine(HybridInstance hybridInstance) {
this.hybridInstance = hybridInstance;
+ this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance());
+ this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance());
}
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
// search the historic realization
- ITupleIterator iterator1 = searchRealization(hybridInstance.getHistoryRealizationInstance(), context, sqlDigest, returnTupleInfo);
+ ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest, returnTupleInfo);
String modelName = hybridInstance.getModelName();
MetadataManager metaMgr = getMetadataManager();
@@ -42,28 +46,32 @@ public class HybridStorageEngine implements IStorageEngine {
// if the model isn't partitioned, only query the history
if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
- return iterator1;
+ return historicalDataIterator;
TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
- ITupleIterator iterator2 = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
+ ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
new Function<Void, ITupleIterator>() {
@Nullable
@Override
public ITupleIterator apply(@Nullable Void input) {
- ITupleIterator iterator2 = searchRealization(hybridInstance.getRealTimeRealizationInstance(), context, sqlDigest, returnTupleInfo);
- return iterator2;
+ return realtimeStorageEngine.search(context, sqlDigest, returnTupleInfo);
}
});
// combine history and real-time tuple iterator
- return new CompoundTupleIterator(Lists.newArrayList(iterator1, iterator2));
+ return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return RangeUtil.merge(historicalStorageEngine.getVolatilePeriod(), realtimeStorageEngine.getVolatilePeriod());
}
- private ITupleIterator searchRealization(IRealization realization, StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(realization, false);
- return storageEngine.search(context, sqlDigest, returnTupleInfo);
+ @Override
+ public boolean isDynamic() {
+ return true;
}
private MetadataManager getMetadataManager() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc426cc0/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index 2a258df..fece398 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -18,12 +18,7 @@
package org.apache.kylin.storage.test;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
+import com.google.common.collect.ImmutableList;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -41,7 +36,11 @@ import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.*;
-import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
public class StorageTest extends HBaseMetadataTestCase {
@@ -64,7 +63,7 @@ public class StorageTest extends HBaseMetadataTestCase {
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
Assert.assertNotNull(cube);
- storageEngine = StorageEngineFactory.getStorageEngine(cube, false);
+ storageEngine = StorageEngineFactory.getStorageEngine(cube);
String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
context = new StorageContext();
context.setConnUrl(url);
[40/50] [abbrv] incubator-kylin git commit: KYLIN-760 Bug fix in
Improve the hasing performance in Sampling cuboid size
Posted by sh...@apache.org.
KYLIN-760 Bug fix in Improve the hasing performance in Sampling cuboid size
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/12bbce6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/12bbce6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/12bbce6f
Branch: refs/heads/streaming-localdict
Commit: 12bbce6f13e6bd7b98967cc10613512f590c9157
Parents: 386e0c4
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed May 13 13:05:21 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed May 13 13:05:35 2015 +0800
----------------------------------------------------------------------
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../apache/kylin/job/cube/CubingJobBuilder.java | 14 ++-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 27 +----
.../job/hadoop/cubev2/SaveStatisticsStep.java | 112 +++++++++++++++++++
4 files changed, 127 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/12bbce6f/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 2e5d97a..6c27cd7 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -55,6 +55,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_BULK_LOAD_HFILE = "Load HFile to HBase Table";
public static final String STEP_NAME_MERGE_DICTIONARY = "Merge Cuboid Dictionary";
public static final String STEP_NAME_MERGE_STATISTICS = "Merge Cuboid Statistics";
+ public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid Statistics";
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/12bbce6f/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index e534441..aca62e8 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -34,6 +34,7 @@ import org.apache.kylin.job.hadoop.cube.*;
import org.apache.kylin.job.hadoop.cubev2.InMemCuboidJob;
import org.apache.kylin.job.hadoop.cubev2.MergeCuboidFromHBaseJob;
import org.apache.kylin.job.hadoop.cubev2.MergeStatisticsStep;
+import org.apache.kylin.job.hadoop.cubev2.SaveStatisticsStep;
import org.apache.kylin.job.hadoop.dict.CreateDictionaryJob;
import org.apache.kylin.job.hadoop.hbase.BulkLoadJob;
import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
@@ -214,9 +215,9 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId);
result.addTask(intermediateHiveTableStep);
result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId));
- result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath));
MapReduceExecutable baseCuboidStep = null;
if (!inMemoryCubing()) {
+ result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath));
// base cuboid step
baseCuboidStep = createBaseCuboidStep(seg, intermediateHiveTableLocation, cuboidOutputTempPath);
result.addTask(baseCuboidStep);
@@ -228,6 +229,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
}
} else {
// create htable step
+ result.addTask(createSaveStatisticsStep(seg, getStatisticsPath(seg, jobId)));
+ result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath));
result.addTask(createCreateHTableStep(seg));
baseCuboidStep = createInMemCubingStep(seg, intermediateHiveTableLocation, intermediateHiveTableName, cuboidOutputTempPath, result.getId());
result.addTask(baseCuboidStep);
@@ -516,6 +519,15 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return result;
}
+ private SaveStatisticsStep createSaveStatisticsStep(CubeSegment seg, String statisticsPath) {
+ SaveStatisticsStep result = new SaveStatisticsStep();
+ result.setName(ExecutableConstants.STEP_NAME_SAVE_STATISTICS);
+ result.setCubeName(seg.getCubeInstance().getName());
+ result.setSegmentId(seg.getUuid());
+ result.setStatisticsPath(statisticsPath);
+ return result;
+ }
+
private MapReduceExecutable createMergeCuboidDataStep(CubeSegment seg, String inputPath, String outputPath) {
MapReduceExecutable mergeCuboidDataStep = new MapReduceExecutable();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/12bbce6f/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index f8863a5..da587db 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -97,14 +97,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
// CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
attachKylinPropsAndMetadata(cubeInstance, job.getConfiguration());
- int result = waitForCompletion(job);
-
- if(result == 0 && Boolean.parseBoolean(statistics_enabled)) {
- putStatisticsToResourceStore(statistics_output, newSegment);
- }
-
- return result;
-
+ return waitForCompletion(job);
} catch (Exception e) {
logger.error("error in FactDistinctColumnsJob", e);
@@ -115,8 +108,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
}
private void setupMapper(String intermediateTable) throws IOException {
-// FileInputFormat.setInputPaths(job, input);
-
String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
HCatInputFormat.setInput(job, dbTableNames[0],
dbTableNames[1]);
@@ -142,22 +133,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
deletePath(job.getConfiguration(), output);
}
- private void putStatisticsToResourceStore(String statisticsFolder, CubeSegment cubeSegment) throws IOException {
- Path statisticsFilePath = new Path(statisticsFolder, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
- FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
- if (!fs.exists(statisticsFilePath))
- throw new IOException("File " + statisticsFilePath + " does not exists;");
-
- FSDataInputStream is = fs.open(statisticsFilePath);
- try {
- // put the statistics to metadata store
- String statisticsFileName = cubeSegment.getStatisticsResourcePath();
- ResourceStore rs = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- rs.putResource(statisticsFileName, is, System.currentTimeMillis());
- } finally {
- IOUtils.closeStream(is);
- }
- }
public static void main(String[] args) throws Exception {
FactDistinctColumnsJob job = new FactDistinctColumnsJob();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/12bbce6f/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/SaveStatisticsStep.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/SaveStatisticsStep.java
new file mode 100644
index 0000000..00fde45
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/SaveStatisticsStep.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cubev2;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.constant.BatchConstants;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Save the cube segment statistic to Kylin metadata store
+ *
+ * @author shaoshi
+ */
+public class SaveStatisticsStep extends AbstractExecutable {
+
+ private static final String CUBE_NAME = "cubeName";
+ private static final String SEGMENT_ID = "segmentId";
+ private static final String STATISTICS_PATH = "statisticsPath";
+
+ public SaveStatisticsStep() {
+ super();
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ KylinConfig kylinConf = context.getConfig();
+ final CubeManager mgr = CubeManager.getInstance(kylinConf);
+ final CubeInstance cube = mgr.getCube(getCubeName());
+ final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
+
+ ResourceStore rs = ResourceStore.getStore(kylinConf);
+ try {
+ Path statisticsFilePath = new Path(getStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
+ if (!fs.exists(statisticsFilePath))
+ throw new IOException("File " + statisticsFilePath + " does not exists;");
+
+ FSDataInputStream is = fs.open(statisticsFilePath);
+ try {
+ // put the statistics to metadata store
+ String statisticsFileName = newSegment.getStatisticsResourcePath();
+ rs.putResource(statisticsFileName, is, System.currentTimeMillis());
+ } finally {
+ IOUtils.closeStream(is);
+ }
+
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+ } catch (IOException e) {
+ logger.error("fail to save cuboid statistics", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+ }
+
+
+ public void setCubeName(String cubeName) {
+ this.setParam(CUBE_NAME, cubeName);
+ }
+
+ private String getCubeName() {
+ return getParam(CUBE_NAME);
+ }
+
+ public void setSegmentId(String segmentId) {
+ this.setParam(SEGMENT_ID, segmentId);
+ }
+
+ private String getSegmentId() {
+ return getParam(SEGMENT_ID);
+ }
+
+ public void setStatisticsPath(String path) {
+ this.setParam(STATISTICS_PATH, path);
+ }
+
+ private String getStatisticsPath() {
+ return getParam(STATISTICS_PATH);
+ }
+
+}
[42/50] [abbrv] incubator-kylin git commit: KYLIN-763 remove author
information
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java b/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
index a83b87c..5ab97c9 100644
--- a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
+++ b/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job;
import org.apache.kylin.job.execution.AbstractExecutable;
/**
- * Created by qianzhou on 12/16/14.
*/
public abstract class BaseTestExecutable extends AbstractExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 0582de3..dc42dc5 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -76,7 +76,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
- * Created by qianzhou on 3/9/15.
*
* This class is going to be deleted
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 48f7a82..65faad7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -80,7 +80,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
/**
- * Created by qianzhou on 3/9/15.
*/
public class BuildIIWithStreamTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index 7b10a22..7161f59 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -42,7 +42,6 @@ import java.util.Map;
import static org.junit.Assert.assertTrue;
/**
- * Created by honma on 6/19/14.
*/
public class DataGenTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java b/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
index 10d08ba..ac4828e 100644
--- a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
+++ b/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 12/22/14.
*/
public class ErrorTestExecutable extends BaseTestExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
index 18173d2..e18cc0d 100644
--- a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
+++ b/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 12/22/14.
*/
public class FailedTestExecutable extends BaseTestExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
index 74b8891..6aaeaa2 100644
--- a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
@@ -49,7 +49,6 @@ import java.io.File;
import java.io.IOException;
/**
- * Created by qianzhou on 3/6/15.
*/
public class IIStreamBuilderTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java b/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index 641d031..b4f6a98 100644
--- a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 12/31/14.
*/
public class SelfStopExecutable extends BaseTestExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
index 861531f..1421f10 100644
--- a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
+++ b/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
@@ -23,7 +23,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
/**
- * Created by qianzhou on 12/22/14.
*/
public class SucceedTestExecutable extends BaseTestExecutable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
index d9a1860..44ba8f4 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
@@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
- * Created by honma on 5/29/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class ColumnConfig {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 7cafea6..5827946 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -36,7 +36,6 @@ import java.text.SimpleDateFormat;
import java.util.*;
/**
- * Created by hongbin on 5/20/14.
*/
public class FactTableGenerator {
CubeInstance cube = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
index b51ddf7..9412944 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
@@ -30,7 +30,6 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import org.apache.kylin.common.util.JsonUtil;
/**
- * Created by honma on 5/29/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class GenConfig {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
index bf915e6..c303ebf 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
@@ -19,7 +19,6 @@ import java.util.Map;
import java.util.Random;
/**
- * Created by Hongbin Ma(Binmahone) on 5/5/15.
*/
public class StreamingDataGenerator {
private static final Logger logger = LoggerFactory.getLogger(StreamingDataGenerator.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
index f321fc3..386d858 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/CubeSamplingTest.java
@@ -34,7 +34,6 @@ import java.util.BitSet;
import java.util.List;
/**
- * Created by shaoshi on 5/12/15.
*/
public class CubeSamplingTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
index 1dc5c44..3232a80 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.kylin.common.util.Bytes;
/**
- * Created by hongbin on 5/15/14.
*/
public class TestHbaseClient {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
index 03a84fb..3b4a6af 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
@@ -32,7 +32,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
/**
- * Created by honma on 8/20/14.
*/
public class HdfsOpsTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index ecc3c23..7a88420 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -54,7 +54,6 @@ import java.io.IOException;
import java.util.*;
/**
- * Created by Hongbin Ma(Binmahone) on 3/26/15.
*/
public class IITest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 9da0f73..f47e152 100644
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -32,7 +32,6 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
/**
- * Created by qianzhou on 12/26/14.
*/
public abstract class BaseSchedulerTest extends HBaseMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index a0ec671..22afb1d 100644
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -31,7 +31,6 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
/**
- * Created by qianzhou on 12/19/14.
*/
public class DefaultSchedulerTest extends BaseSchedulerTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java b/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
index 11af88e..1d28fc8 100644
--- a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
@@ -37,7 +37,6 @@ import java.util.List;
import static org.junit.Assert.*;
/**
- * Created by qianzhou on 12/16/14.
*/
public class ExecutableManagerTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
index 9587e89..95e4b7f 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
@@ -19,7 +19,6 @@ import org.junit.Ignore;
import org.junit.Test;
/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
*/
@Ignore
public class HBaseRowDigestTest extends HBaseMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
index 74e2d71..8d5c10c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
@@ -11,7 +11,6 @@ import java.util.List;
import java.util.Set;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class DateConditionModifier implements TupleFilterSerializer.Decorator {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
index 5a7ee40..96ff81c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
@@ -6,7 +6,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import java.util.IdentityHashMap;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class TsConditionEraser implements TupleFilterSerializer.Decorator {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
index 138940f..49cd327 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -6,7 +6,6 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.model.DataType;
/**
- * Created by Hongbin Ma(Binmahone) on 2/10/15.
*/
public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
index 7f26482..73e71b6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DimensionDesc.java
@@ -25,7 +25,6 @@ import org.apache.kylin.common.util.StringUtil;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 12/26/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class DimensionDesc {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
index abf87b7..e1ab0af 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
@@ -21,7 +21,6 @@ package org.apache.kylin.metadata.model;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
*/
public interface IJoinedFlatTableDesc {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/model/IntermediateColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/IntermediateColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/IntermediateColumnDesc.java
index 1b931a0..9dbe63e 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/IntermediateColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/IntermediateColumnDesc.java
@@ -21,7 +21,6 @@ package org.apache.kylin.metadata.model;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
*/
public class IntermediateColumnDesc {
private String id;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
index cfbe6a3..910d242 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.kylin.metadata.realization.RealizationType;
/**
- * Created by qianzhou on 12/5/14.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class RealizationEntry {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index c38f5da..3a07cc6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -1,7 +1,6 @@
package org.apache.kylin.metadata.realization;
/**
- * Created by Hongbin Ma(Binmahone) on 2/5/15.
*/
public class IRealizationConstants {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
index 2c7ba42..618dff5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
@@ -34,7 +34,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by Hongbin Ma(Binmahone) on 12/18/14.
*/
public class RealizationRegistry {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
index 1b5dbc5..3fcfe63 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
@@ -19,7 +19,6 @@
package org.apache.kylin.metadata.realization;
/**
- * Created by qianzhou on 12/8/14.
*/
//TODO: change to String for plugin
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index f31aa97..7811858 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -26,7 +26,6 @@ import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
*/
public class SQLDigest {
public String factTable;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index 695ac8f..19b9ee7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -10,7 +10,6 @@ import com.google.common.collect.BoundType;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/14/15.
*/
public class SQLDigestUtil {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
index 3a1d41a..4fd9f17 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
@@ -8,7 +8,6 @@ import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*
* A encapsulation of {@link SQLDigest},
* This class makes {@link SQLDigest} being able to compare with other {@link SQLDigest}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
index 7803d0d..b87c633 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
@@ -10,7 +10,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Created by Hongbin Ma(Binmahone) on 4/14/15.
*/
public class CompoundTupleIterator implements ITupleIterator {
private static final Logger logger = LoggerFactory.getLogger(CompoundTupleIterator.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
index bbad038..ebdcb5d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/SimpleTupleIterator.java
@@ -5,7 +5,6 @@ import java.util.Iterator;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/14/15.
*
*/
public class SimpleTupleIterator implements ITupleIterator {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
index 9a6a2be..556fca6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
@@ -7,7 +7,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/15/15.
*
* Like "tee" command in linux, it effectively duplicates the underlying
* ITupleIterator's results
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java b/metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
index b0e2b5a..bb30c1e 100644
--- a/metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
+++ b/metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
@@ -4,7 +4,6 @@ import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
/**
- * Created by Hongbin Ma(Binmahone) on 5/4/15.
*/
public class FixedPointLongCodecTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/metadata/src/test/java/org/apache/kylin/metadata/model/TableDescTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/model/TableDescTest.java b/metadata/src/test/java/org/apache/kylin/metadata/model/TableDescTest.java
index 78895c8..3a69e5c 100644
--- a/metadata/src/test/java/org/apache/kylin/metadata/model/TableDescTest.java
+++ b/metadata/src/test/java/org/apache/kylin/metadata/model/TableDescTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.metadata.model;
/**
- * Created by qianzhou on 12/3/14.
*/
public class TableDescTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index ff717a2..a24bc4e 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -31,7 +31,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.relnode.OLAPContext;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public abstract class RoutingRule {
private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
index 3f3a5d9..eba309a 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
@@ -37,7 +37,6 @@ import java.util.Iterator;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class AdjustForWeaklyMatchedRealization extends RoutingRule {
private static final Logger logger = LoggerFactory.getLogger(AdjustForWeaklyMatchedRealization.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
index 96fafde..4389263 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
@@ -27,7 +27,6 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RoutingRule;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class RealizationPriorityRule extends RoutingRule {
static Map<RealizationType, Integer> priorities = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
index ed8644f..f9a250e 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
@@ -28,7 +28,6 @@ import java.util.Comparator;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class RealizationSortRule extends RoutingRule {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
index fc68ddd..c710ff1 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
@@ -26,7 +26,6 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RoutingRule;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class RemoveUncapableRealizationsRule extends RoutingRule {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
index 7a3128b..9e8e92b 100644
--- a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
@@ -29,7 +29,6 @@ import java.util.Arrays;
import java.util.Collection;
/**
- * Created by honma on 7/2/14.
*/
@RunWith(Parameterized.class)
public class CombinationTest extends KylinQueryTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index 37fc4f3..185b22e 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -32,7 +32,6 @@ import java.util.Collection;
import java.util.Map;
/**
- * Created by Hongbin Ma(Binmahone) on 2/2/15.
*/
@RunWith(Parameterized.class)
public class IIQueryTest extends KylinQueryTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
index 89fb01d..0f5c1ec 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
@@ -41,7 +41,6 @@ import java.util.*;
import java.util.logging.LogManager;
/**
- * Created by hongbin on 2/18/14.
*/
public class KylinTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/query/src/test/java/org/apache/kylin/query/test/RealizationRegistryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/RealizationRegistryTest.java b/query/src/test/java/org/apache/kylin/query/test/RealizationRegistryTest.java
index dc814ea..c1789db 100644
--- a/query/src/test/java/org/apache/kylin/query/test/RealizationRegistryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/RealizationRegistryTest.java
@@ -32,7 +32,6 @@ import org.junit.Before;
import org.junit.Test;
/**
- * Created by qianzhou on 1/26/15.
*/
public class RealizationRegistryTest extends LocalFileMetadataTestCase {
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java b/server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java
index bcf5ad5..7ffea38 100644
--- a/server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java
+++ b/server/src/main/java/org/apache/kylin/rest/model/ColumnMeta.java
@@ -21,7 +21,6 @@ package org.apache.kylin.rest.model;
import java.io.Serializable;
/**
- * Created by lukhan on 2/27/14.
*/
public class ColumnMeta implements Serializable {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java b/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
index 269fbab..00b01db 100644
--- a/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
+++ b/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.model;
/**
- * Created by lukhan on 2/27/14.
*/
public class SelectedColumnMeta {
public SelectedColumnMeta(boolean isAutoIncrement, boolean isCaseSensitive, boolean isSearchable, boolean isCurrency, int isNullalbe, boolean isSigned, int displaySize, String label, String name, String schemaName, String catelogName, String tableName, int precision, int scale, int columnType, String columnTypeName, boolean isReadOnly, boolean isWritable, boolean isDefinitelyWritable) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/model/TableMeta.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/model/TableMeta.java b/server/src/main/java/org/apache/kylin/rest/model/TableMeta.java
index 36f7aa1..485e745 100644
--- a/server/src/main/java/org/apache/kylin/rest/model/TableMeta.java
+++ b/server/src/main/java/org/apache/kylin/rest/model/TableMeta.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.List;
/**
- * Created by lukhan on 2/27/14.
*/
public class TableMeta implements Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java b/server/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
index 075fa52..71cd1c4 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.request;
/**
- * Created by honma on 8/7/14.
*/
public class CreateProjectRequest {
private String name;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/request/MetaRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/MetaRequest.java b/server/src/main/java/org/apache/kylin/rest/request/MetaRequest.java
index 9ec9c1d..66e8ec0 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/MetaRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/MetaRequest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.request;
/**
- * Created by honma on 7/10/14.
*/
public class MetaRequest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java b/server/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
index 3819de6..29ba162 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.request;
/**
- * Created by honma on 8/7/14.
*/
public class UpdateProjectRequest {
private String formerProjectName;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index e104897..6ce8b83 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -32,7 +32,6 @@ import java.io.IOException;
import java.util.List;
/**
- * Created by qianzhou on 1/19/15.
*/
@Component("cacheService")
public class CacheService extends BasicService {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
index 2b1de23..2eff08f 100644
--- a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
+++ b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
@@ -13,7 +13,6 @@ import java.util.List;
import java.util.Properties;
/**
- * Created by shaoshi on 2/5/15.
*/
public class JDBCDriverTest extends HBaseMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java b/server/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
index 2c7e068..ed63183 100644
--- a/server/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
+++ b/server/src/test/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
@@ -28,7 +28,6 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
- * Created by qianzhou on 1/16/15.
*/
public class BroadcasterReceiveServlet extends HttpServlet {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
index 7bcaba8..467b7cd 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
@@ -35,7 +35,6 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
/**
- * Created by honma on 8/7/14.
*/
public class ProjectControllerTest extends ServiceTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 00f521b..64a71c4 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -49,7 +49,6 @@ import java.util.concurrent.atomic.AtomicLong;
import static org.junit.Assert.*;
/**
- * Created by qianzhou on 1/16/15.
*/
public class CacheServiceTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
index 3e31898..068619e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
@@ -42,7 +42,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class CacheFledgedStorageEngine implements IStorageEngine {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java b/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
index 18a0e02..9b4caa5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
@@ -1,7 +1,6 @@
package org.apache.kylin.storage.cache;
/**
- * Created by Hongbin Ma(Binmahone) on 5/11/15.
*/
public interface StorageLayerCache {
boolean isCacheEnabled();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 52405d8..5e58822 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -19,7 +19,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class StreamSQLResult {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
index 9b7550f..4ca8e78 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
@@ -21,7 +21,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Created by shaoshi on 3/23/15.
* This implementation uses Dictionary to encode and decode the table; If a column doesn't have dictionary, will check
* its data type to serialize/deserialize it;
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
index a6476ca..f07b516 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -10,7 +10,6 @@ import java.io.IOException;
import java.util.BitSet;
/**
- * Created by qianzhou on 5/6/15.
*/
public class GTComboStore implements IGTStore {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
index 5c0c436..d8edf45 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
@@ -5,7 +5,6 @@ import org.apache.kylin.common.util.ByteArray;
import java.util.BitSet;
/**
- * Created by qianzhou on 5/6/15.
*/
public final class ScanKey {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
index bebc1a2..db245fd 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
@@ -4,7 +4,6 @@ import java.io.InputStream;
import java.io.OutputStream;
/**
- * Created by qianzhou on 5/4/15.
*/
interface FileSystem {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
index fec0d13..25d66f0 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -16,7 +16,6 @@ import java.util.BitSet;
import java.util.UUID;
/**
- * Created by qianzhou on 5/4/15.
*/
public class GTDiskStore implements IGTStore {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
index 7a6450d..9546db7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
@@ -10,7 +10,6 @@ import java.io.InputStream;
import java.io.OutputStream;
/**
- * Created by qianzhou on 5/6/15.
*/
class HadoopFileSystem implements FileSystem {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
index d512552..0d07f3b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
@@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory;
import java.io.*;
/**
- * Created by qianzhou on 5/4/15.
*/
class LocalFileSystem implements FileSystem {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
index 20deaeb..ae62959 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
@@ -9,7 +9,6 @@ import org.apache.kylin.cube.kv.RowConstants;
import com.google.common.collect.Lists;
/**
- * Created by qianzhou on 4/20/15.
*/
public class AggrKey implements Comparable<AggrKey> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
index f951409..4274b9d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
@@ -25,7 +25,6 @@ import org.apache.kylin.metadata.measure.MeasureAggregator;
import com.google.common.collect.Maps;
/**
- * Created by Hongbin Ma(Binmahone) on 11/27/14.
*/
@SuppressWarnings("rawtypes")
public abstract class AggregationCache {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
index 7efb283..6e69827 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
@@ -19,7 +19,6 @@
package org.apache.kylin.storage.hbase.coprocessor;
/**
- * Created by Hongbin Ma(Binmahone) on 11/26/14.
*/
public class CoprocessorConstants {
public static final int SERIALIZE_BUFFER_SIZE = 65536;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
index 73bfd25..7729e8c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
@@ -13,7 +13,6 @@ import java.util.Collection;
import java.util.Set;
/**
- * Created by Hongbin Ma(Binmahone) on 3/3/15.
*/
@SuppressWarnings("unchecked")
public class FilterDecorator implements TupleFilterSerializer.Decorator {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
index e93bf17..e9a3780 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
@@ -45,7 +45,6 @@ import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import java.util.Map;
/**
- * Created by qianzhou on 3/17/15.
*/
public class ClearTextDictionary implements IDictionaryAware {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
index ac6fa70..3ab3934 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
@@ -26,7 +26,6 @@ import java.util.Map;
import java.util.Set;
/**
- * Created by Hongbin Ma(Binmahone) on 11/27/14.
*/
public class EndpointAggregationCache extends AggregationCache {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 6a7c14a..d9ede57 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -58,7 +58,6 @@ import java.io.IOException;
import java.util.*;
/**
- * Created by Hongbin Ma(Binmahone) on 12/2/14.
*/
public class EndpointTupleIterator implements ITupleIterator {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 4c1c568..5952a1f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -33,7 +33,6 @@ import java.util.List;
import java.util.NoSuchElementException;
/**
- * Created by honma on 11/10/14.
*/
public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index af8d917..429e060 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -64,7 +64,6 @@ import com.google.protobuf.Service;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
/**
- * Created by honma on 11/7/14.
*/
public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, CoprocessorService {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIResponseAdapter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIResponseAdapter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIResponseAdapter.java
index eb3eef3..8675de1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIResponseAdapter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIResponseAdapter.java
@@ -19,7 +19,6 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
/**
- * Created by honma on 11/10/14.
*/
public class IIResponseAdapter {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
index 45de6fe..0848a5c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/LocalDictionary.java
@@ -9,7 +9,6 @@ import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import java.util.Map;
/**
- * Created by Hongbin Ma(Binmahone) on 3/3/15.
*/
public class LocalDictionary implements IDictionaryAware {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
index 0691560..09327eb 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java
@@ -26,7 +26,6 @@ import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
/**
- * Created by Hongbin Ma(Binmahone) on 11/24/14.
* <p/>
* an adapter
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
index f16e40c..18155c6 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
@@ -10,7 +10,6 @@ import org.apache.kylin.common.util.DateFormat;
import com.google.common.collect.Range;
/**
- * Created by Hongbin Ma(Binmahone) on 4/8/15.
*/
public class TsConditionExtractor {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 0350489..beb86c5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -18,7 +18,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import java.util.List;
/**
- * Created by shaoshi on 2/13/15.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 7e4621c..df3fae3 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -18,7 +18,6 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by shaoshi on 2/13/15.
*/
public class HybridManager implements IRealizationProvider {
public static final Serializer<HybridInstance> HYBRID_SERIALIZER = new JsonSerializer<HybridInstance>(HybridInstance.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 96b6ad0..87ff26b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -21,7 +21,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Ranges;
/**
- * Created by shaoshi on 2/13/15.
*/
public class HybridStorageEngine implements IStorageEngine {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java b/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
index d390415..2ac2445 100644
--- a/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
@@ -3,7 +3,6 @@ package org.apache.kylin.storage.util;
import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
/**
- * Created by qianzhou on 5/11/15.
*/
public final class SizeOfUtil {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
index fd34224..e86771d 100644
--- a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -15,7 +15,6 @@ import java.util.HashMap;
import java.util.HashSet;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class EhcacheTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
index 29cf7b1..669289e 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
@@ -9,7 +9,6 @@ import org.junit.Assert;
import org.junit.Test;
/**
- * Created by Hongbin Ma(Binmahone) on 5/7/15.
*/
public class DateConditionModifierTest extends FilterBaseTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
index c1fff70..59b1af0 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
@@ -20,7 +20,6 @@ import org.junit.Test;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
*/
public class TsConditionEraserTest extends LocalFileMetadataTestCase {
IIInstance ii;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
index ae0a9bf..3217f1c 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
@@ -43,7 +43,6 @@ import org.junit.Test;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone) on 11/27/14.
*
* ii test
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
index 4369b26..df1c26b 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
@@ -32,7 +32,6 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
/**
- * Created by Hongbin Ma(Binmahone) on 12/3/14.
*/
public class TableRecordInfoTest extends LocalFileMetadataTestCase {
IIInstance ii;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
index 99ce8b9..7a04ee6 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
@@ -38,7 +38,6 @@ import org.junit.Test;
import com.google.common.collect.Lists;
/**
- * Created by Hongbin Ma(Binmahone)
*
* ii test
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 775ef9d..3ad7ca6 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -11,7 +11,6 @@ import org.junit.Before;
import org.junit.Test;
/**
- * Created by shaoshi on 2/13/15.
*/
public class HybridManagerTest extends LocalFileMetadataTestCase {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/BrokerConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/BrokerConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/BrokerConfig.java
index f229202..f1104e1 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/BrokerConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/BrokerConfig.java
@@ -38,7 +38,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
- * Created by qianzhou on 3/25/15.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class BrokerConfig {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
index d229647..7d730b7 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
@@ -50,7 +50,6 @@ import java.util.List;
import java.util.Map;
/**
- * Created by qianzhou on 3/25/15.
*/
public final class JsonStreamParser implements StreamParser {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConfig.java
index 77dff2b..9949c96 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConfig.java
@@ -48,7 +48,6 @@ import java.io.*;
import java.util.List;
/**
- * Created by qianzhou on 3/2/15.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class KafkaConfig extends RootPersistentEntity {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
index 715eefe..70d37ef 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaConsumer.java
@@ -49,7 +49,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/**
- * Created by qianzhou on 2/15/15.
*/
public class KafkaConsumer implements Runnable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
index 62f82a2..02e2759 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
@@ -54,7 +54,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentMap;
/**
- * Created by qianzhou on 2/15/15.
*/
public final class KafkaRequester {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
index 4f5f0f5..67cb109 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamBuilder.java
@@ -43,7 +43,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
- * Created by qianzhou on 2/17/15.
*/
public abstract class StreamBuilder implements Runnable {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/StreamMessage.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamMessage.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamMessage.java
index 4747b7d..6229aa5 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamMessage.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamMessage.java
@@ -35,7 +35,6 @@
package org.apache.kylin.streaming;
/**
- * Created by qianzhou on 2/15/15.
*/
public class StreamMessage {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/StreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamParser.java
index afbfff0..304e252 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamParser.java
@@ -37,7 +37,6 @@ package org.apache.kylin.streaming;
import java.util.List;
/**
- * Created by qianzhou on 3/25/15.
*/
public interface StreamParser {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/StreamingManager.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingManager.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingManager.java
index d3a7c70..46c34fa 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingManager.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingManager.java
@@ -44,7 +44,6 @@ import java.io.*;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by qianzhou on 3/25/15.
*/
public class StreamingManager {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/StringStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StringStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/StringStreamParser.java
index 8f99eb5..14e1b45 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StringStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StringStreamParser.java
@@ -39,7 +39,6 @@ import java.util.List;
import com.google.common.collect.Lists;
/**
- * Created by qianzhou on 3/25/15.
*/
public final class StringStreamParser implements StreamParser {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java b/streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java
index b93d589..4338b18 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/TopicMeta.java
@@ -40,7 +40,6 @@ import java.util.List;
/**
* The topic metadata should be invariant, otherwise will cause re-initialization of the Consumer
*
- * Created by qianzhou on 2/15/15.
*/
public class TopicMeta {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 7c1d435..755b490 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -57,7 +57,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
- * Created by qianzhou on 3/3/15.
*/
public class IIStreamBuilder extends StreamBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/SliceBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/SliceBuilder.java
index 7ca374f..b6769a9 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/SliceBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/SliceBuilder.java
@@ -52,7 +52,6 @@ import javax.annotation.Nullable;
import java.util.List;
/**
- * Created by qianzhou on 3/27/15.
*/
public final class SliceBuilder {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java
index 3919335..30b7588 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaBaseTest.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
- * Created by qianzhou on 2/16/15.
*/
public abstract class KafkaBaseTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
index cb32030..c782dae 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
@@ -50,7 +50,6 @@ import java.util.concurrent.Executors;
import static org.junit.Assert.assertTrue;
/**
- * Created by qianzhou on 2/16/15.
*/
public class KafkaConsumerTest extends KafkaBaseTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
index fb6ed84..9d99825 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
@@ -43,7 +43,6 @@ import org.junit.Test;
import static org.junit.Assert.*;
/**
- * Created by qianzhou on 2/16/15.
*/
public class KafkaRequesterTest extends KafkaBaseTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc056fda/streaming/src/test/java/org/apache/kylin/streaming/OneOffStreamProducer.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/OneOffStreamProducer.java b/streaming/src/test/java/org/apache/kylin/streaming/OneOffStreamProducer.java
index dcd9642..c17a3fc 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/OneOffStreamProducer.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/OneOffStreamProducer.java
@@ -53,7 +53,6 @@ import java.io.IOException;
import java.util.Properties;
/**
- * Created by qianzhou on 2/16/15.
*/
public class OneOffStreamProducer {
[48/50] [abbrv] incubator-kylin git commit: KYLIN-759 add test cases
and bug fix
Posted by sh...@apache.org.
KYLIN-759 add test cases and bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0e42b2df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0e42b2df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0e42b2df
Branch: refs/heads/streaming-localdict
Commit: 0e42b2dfae6f490ae25e33241f3982f7de66fd59
Parents: bc426cc
Author: honma <ho...@ebay.com>
Authored: Thu May 14 13:57:31 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 14 20:22:04 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/IdentityUtils.java | 35 +++++
.../org/apache/kylin/common/util/BasicTest.java | 24 ++--
.../kylin/common/util/IdentityUtilTest.java | 30 ++++
.../apache/kylin/query/test/KylinQueryTest.java | 2 +-
.../kylin/storage/ICachableStorageEngine.java | 7 +
.../cache/CacheFledgedDynamicStorageEngine.java | 5 +-
.../cache/CacheFledgedStaticStorageEngine.java | 2 +-
.../kylin/storage/cache/StreamSQLResult.java | 3 +
.../org/apache/kylin/storage/tuple/Tuple.java | 8 +-
.../apache/kylin/storage/cache/EhcacheTest.java | 5 -
.../kylin/storage/test/DynamicCacheTest.java | 141 +++++++++++++++++++
.../kylin/storage/test/StaticCacheTest.java | 95 +++++++++++++
.../kylin/storage/test/StorageMockUtils.java | 132 +++++++++++++++++
.../apache/kylin/storage/test/StorageTest.java | 119 ++--------------
14 files changed, 476 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/common/src/main/java/org/apache/kylin/common/util/IdentityUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/IdentityUtils.java b/common/src/main/java/org/apache/kylin/common/util/IdentityUtils.java
new file mode 100644
index 0000000..d0691da
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/IdentityUtils.java
@@ -0,0 +1,35 @@
+package org.apache.kylin.common.util;
+
+import java.util.Collection;
+import java.util.IdentityHashMap;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public class IdentityUtils {
+ public static <K> boolean collectionReferenceEquals(Collection<K> collectionA, Collection<K> collectionB) {
+ if (collectionA == null || collectionB == null) {
+ throw new RuntimeException("input must be not null");
+ }
+
+ IdentityHashMap<K, Void> mapA = new IdentityHashMap<>();
+ IdentityHashMap<K, Void> mapB = new IdentityHashMap<>();
+ for (K key : collectionA) {
+ mapA.put(key, null);
+ }
+ for (K key : collectionB) {
+ mapB.put(key, null);
+ }
+
+ if (mapA.keySet().size() != mapB.keySet().size()) {
+ return false;
+ }
+
+ for (K key : mapA.keySet()) {
+ if (!mapB.keySet().contains(key)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index e0226f8..b8fb5e7 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -19,8 +19,8 @@
package org.apache.kylin.common.util;
import com.google.common.collect.Lists;
-import com.google.common.collect.Ranges;
import org.apache.commons.configuration.ConfigurationException;
+import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.LoggerFactory;
@@ -31,6 +31,7 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
+import java.util.IdentityHashMap;
/**
* <p/>
@@ -60,16 +61,21 @@ public class BasicTest {
}
@Test
+ public void test0() throws Exception {
+
+ IdentityHashMap<String, Void> a = new IdentityHashMap<>();
+ IdentityHashMap<String, Void> b = new IdentityHashMap<>();
+ String s1 = new String("s1");
+ String s2 = new String("s1");
+ Assert.assertEquals(s1, s2);
+ Assert.assertTrue(s1 != s2);
+ a.put(s1, null);
+ b.put(s2, null);
+ }
+
+ @Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
- Number xx =new Long(0L);
- System.out.println(xx.getClass().getName());
-
- System.out.println(time(1367798400000L));
-
- System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1000L * Integer.MAX_VALUE));
-
- System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(3, 10)));
String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12 07:59:37
String cc = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/10/16 08:00:00
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
new file mode 100644
index 0000000..99385f6
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/IdentityUtilTest.java
@@ -0,0 +1,30 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public class IdentityUtilTest {
+ @Test
+ public void basicTest()
+ {
+ String s1 = new String("hi");
+ String s2 = new String("hi");
+
+ List<String> c1 = Lists.newArrayList(s1);
+ List<String> c2 = Lists.newArrayList(s2);
+ List<String> c3 = Lists.newArrayList(s2);
+
+ Assert.assertTrue(CollectionUtils.isEqualCollection(c1,c2));
+ Assert.assertTrue(CollectionUtils.isEqualCollection(c3,c2));
+
+ Assert.assertFalse(IdentityUtils.collectionReferenceEquals(c1,c2));
+ Assert.assertTrue(IdentityUtils.collectionReferenceEquals(c3,c2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 40372ff..cea8ee2 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -95,7 +95,7 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql_derived/query07.sql";
+ String queryFileName = "src/test/resources/query/sql/query00.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
new file mode 100644
index 0000000..eadef09
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
@@ -0,0 +1,7 @@
+package org.apache.kylin.storage;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public interface ICachableStorageEngine extends IStorageEngine{
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
index 0d7ab18..65672ae 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
@@ -41,7 +41,8 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
- //if isDynamicStorage && query involves filter on partition column, the cache requires updating after query done.
+ //enable dynamic cache iff group by columns contains partition col
+ //because cache extraction requires partition col value as selection key
boolean needUpdateCache = sqlDigest.groupbyColumns.contains(partitionColRef);
streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
@@ -106,7 +107,7 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
logger.info("decision: use cache");
}
- if (needUpdateCache || !queryCacheExists) {
+ if (needUpdateCache) {
//use another nested ITupleIterator to deal with cache
final TeeTupleIterator tee = new TeeTupleIterator(ret);
tee.addCloseListener(this);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
index a4aceed..860dea2 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
@@ -51,7 +51,7 @@ public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorage
logger.info("decision: use cache");
}
- if (queryCacheExists) {
+ if (!queryCacheExists) {
//use another nested ITupleIterator to deal with cache
final TeeTupleIterator tee = new TeeTupleIterator(ret);
tee.addCloseListener(this);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index e895d8e..924044b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -39,6 +39,9 @@ public class StreamSQLResult {
}
}
} else {
+ if (!this.sortedRows.containsKey(0L)) {
+ this.sortedRows.put(0L, Lists.<ITuple> newArrayList());
+ }
this.sortedRows.get(0L).add(row);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 7224cfe..efa1bc2 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -135,7 +135,9 @@ public class Tuple implements ITuple {
return 1L * days * (1000 * 3600 * 24);
}
- public static int millisToEpicDays(long millis) {
+ public static int dateToEpicDays(String strValue) {
+ Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
+ long millis = dateValue.getTime();
return (int) (millis / (1000 * 3600 * 24));
}
@@ -158,9 +160,7 @@ public class Tuple implements ITuple {
// TODO use data type enum instead of string comparison
if ("date".equals(dataTypeName)) {
// convert epoch time
- Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
- long millis = dateValue.getTime();
- return millisToEpicDays(millis);// Optiq expects Integer instead of Long. by honma
+ return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
} else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
return Long.parseLong(strValue);
} else if ("tinyint".equals(dataTypeName)) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
index e86771d..1dec9e6 100644
--- a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -1,9 +1,7 @@
package org.apache.kylin.storage.cache;
-import com.google.common.collect.Maps;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.MemoryUnit;
import net.sf.ehcache.config.PersistenceConfiguration;
@@ -11,9 +9,6 @@ import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.HashSet;
-
/**
*/
public class EhcacheTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
new file mode 100644
index 0000000..0d962cb
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
@@ -0,0 +1,141 @@
+package org.apache.kylin.storage.test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.IdentityUtils;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.cache.CacheFledgedDynamicStorageEngine;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public class DynamicCacheTest {
+
+ class TsOnlyTuple implements ITuple {
+ private TblColRef partitionCol;
+ private String tsStr;
+
+ public TsOnlyTuple(TblColRef partitionCol, String tsStr) {
+ this.partitionCol = partitionCol;
+ this.tsStr = tsStr;
+ }
+
+ @Override
+ public List<String> getAllFields() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public List<TblColRef> getAllColumns() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object[] getAllValues() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public ITuple makeCopy() {
+ return new TsOnlyTuple(this.partitionCol, this.tsStr);
+ }
+
+ @Override
+ public Object getValue(TblColRef col) {
+ if (col.equals(partitionCol)) {
+ return Tuple.dateToEpicDays(this.tsStr);
+ } else {
+ throw new NotImplementedException();
+ }
+ }
+ }
+
+ @Test
+ public void basicTest() {
+
+ final StorageContext context = new StorageContext();
+ final List<TblColRef> groups = StorageMockUtils.buildGroups();
+ final TblColRef partitionCol = groups.get(0);
+ final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
+
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef>newArrayList(), aggregations);
+
+ ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01");
+ ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01");
+ final List<ITuple> allTuples = Lists.newArrayList(aTuple, bTuple);
+
+ //counts for verifying
+ final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
+ final List<Integer> returnedRowPerSearch = Lists.newArrayList();
+
+ CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new IStorageEngine() {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter);
+ List<ITuple> ret = Lists.newArrayList();
+ for (ITuple tuple : allTuples) {
+ if (tsRagneInQuery.contains(Tuple.getTs(tuple, partitionCol))) {
+ ret.add(tuple);
+ }
+ }
+
+ underlyingSEHitCount.incrementAndGet();
+ returnedRowPerSearch.add(ret.size());
+
+ return new SimpleTupleIterator(ret.iterator());
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return true;
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01"));
+ }
+ }, partitionCol);
+
+ sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0));
+ ITupleIterator firstIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
+ while (firstIterator.hasNext()) {
+ firstResults.put(firstIterator.next(), null);
+ }
+ firstIterator.close();
+
+ sqlDigest.filter = StorageMockUtils.buildTs2011Filter(groups.get(0));
+ ITupleIterator secondIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
+ while (secondIterator.hasNext()) {
+ secondResults.put(secondIterator.next(), null);
+ }
+ secondIterator.close();
+
+ Assert.assertEquals(2, firstResults.size());
+ IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
+ Assert.assertEquals(2, underlyingSEHitCount.get());
+ Assert.assertEquals(new Integer(2), returnedRowPerSearch.get(0));
+ Assert.assertEquals(new Integer(1), returnedRowPerSearch.get(1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
new file mode 100644
index 0000000..9b69830
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
@@ -0,0 +1,95 @@
+package org.apache.kylin.storage.test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import org.apache.kylin.common.util.IdentityUtils;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.cache.CacheFledgedStaticStorageEngine;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public class StaticCacheTest {
+
+ @Test
+ public void basicTest() {
+
+ final StorageContext context = new StorageContext();
+ final List<TblColRef> groups = StorageMockUtils.buildGroups();
+ final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+ final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
+ final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
+
+ final List<ITuple> ret = Lists.newArrayList();
+ ret.add(new Tuple(tupleInfo));
+ ret.add(new Tuple(tupleInfo));
+ ret.add(new Tuple(tupleInfo));
+
+ final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
+
+ CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new IStorageEngine() {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ underlyingSEHitCount.incrementAndGet();
+ return new SimpleTupleIterator(ret.iterator());
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return null;
+ }
+ });
+
+ ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
+ while (firstIterator.hasNext()) {
+ firstResults.put(firstIterator.next(), null);
+ }
+ firstIterator.close();
+
+ ITupleIterator secondIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple,Void> secondResults = new IdentityHashMap<>();
+ while(secondIterator.hasNext())
+ {
+ secondResults.put(secondIterator.next(),null);
+ }
+ secondIterator.close();
+
+ ITupleIterator thirdIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple,Void> thirdResults = new IdentityHashMap<>();
+ while(thirdIterator.hasNext())
+ {
+ thirdResults.put(thirdIterator.next(),null);
+ }
+ thirdIterator.close();
+
+ Assert.assertEquals(3, firstResults.size());
+ IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
+ IdentityUtils.collectionReferenceEquals(thirdResults.keySet(), secondResults.keySet());
+
+ Assert.assertEquals(1, underlyingSEHitCount.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
new file mode 100644
index 0000000..d6a6da1
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
@@ -0,0 +1,132 @@
+package org.apache.kylin.storage.test;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.storage.tuple.TupleInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/14/15.
+ */
+public class StorageMockUtils {
+ public static TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
+ TupleInfo info = new TupleInfo();
+ int idx = 0;
+
+ for (TblColRef col : groups) {
+ info.setField(col.getName(), col, idx++);
+ }
+
+ TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
+ for (FunctionDesc func : aggregations) {
+ TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
+ info.setField(col.getName(), col, idx++);
+ }
+
+ return info;
+ }
+
+ public static List<TblColRef> buildGroups() {
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef cf1 = new TblColRef(c1);
+ groups.add(cf1);
+
+ TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
+ ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
+ TblColRef cf2 = new TblColRef(c2);
+ groups.add(cf2);
+
+ return groups;
+ }
+
+ public static List<FunctionDesc> buildAggregations() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
+
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ TblColRef priceCol = new TblColRef(ColumnDesc.mockup(t1, 7, "PRICE", "decimal(19,4)"));
+ TblColRef sellerCol = new TblColRef(ColumnDesc.mockup(t1, 9, "SELLER_ID", "bigint"));
+
+ FunctionDesc f1 = new FunctionDesc();
+ f1.setExpression("SUM");
+ ParameterDesc p1 = new ParameterDesc();
+ p1.setType("column");
+ p1.setValue("PRICE");
+ p1.setColRefs(ImmutableList.of(priceCol));
+ f1.setParameter(p1);
+ functions.add(f1);
+
+ FunctionDesc f2 = new FunctionDesc();
+ f2.setExpression("COUNT_DISTINCT");
+ ParameterDesc p2 = new ParameterDesc();
+ p2.setType("column");
+ p2.setValue("SELLER_ID");
+ p2.setColRefs(ImmutableList.of(sellerCol));
+ f2.setParameter(p2);
+ functions.add(f2);
+
+ return functions;
+ }
+
+
+ public static CompareTupleFilter buildTs2010Filter(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2010-01-01");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildTs2011Filter(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-01-01");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildFilter1(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildFilter2(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter2);
+ ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
+ compareFilter.addChild(constantFilter2);
+ return compareFilter;
+ }
+
+ @SuppressWarnings("unused")
+ public static TupleFilter buildAndFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
+ CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter.addChild(compareFilter1);
+ andFilter.addChild(compareFilter2);
+ return andFilter;
+ }
+
+ @SuppressWarnings("unused")
+ public static TupleFilter buildOrFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
+ CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
+ LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ logicFilter.addChild(compareFilter1);
+ logicFilter.addChild(compareFilter2);
+ return logicFilter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0e42b2df/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index fece398..f9a6212 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -18,14 +18,13 @@
package org.apache.kylin.storage.test;
-import com.google.common.collect.ImmutableList;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.filter.*;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
@@ -33,10 +32,8 @@ import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageEngineFactory;
import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.*;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -78,17 +75,17 @@ public class StorageTest extends HBaseMetadataTestCase {
@Ignore
public void testScanOutOfLimit() {
context.setThreshold(1);
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
search(groups, aggregations, null, context);
}
@Test
public void test01() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildFilter1(groups.get(0));
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
int count = search(groups, aggregations, filter, context);
assertTrue(count > 0);
@@ -139,7 +136,7 @@ public class StorageTest extends HBaseMetadataTestCase {
ITupleIterator iterator = null;
try {
SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
- iterator = storageEngine.search(context, sqlDigest, newTupleInfo(groups, aggregations));
+ iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
System.out.println("Tuple = " + tuple);
@@ -154,102 +151,4 @@ public class StorageTest extends HBaseMetadataTestCase {
return count;
}
- private TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
- TupleInfo info = new TupleInfo();
- int idx = 0;
-
- for (TblColRef col : groups) {
- info.setField(col.getName(), col, idx++);
- }
-
- TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
- for (FunctionDesc func : aggregations) {
- TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
- info.setField(col.getName(), col, idx++);
- }
-
- return info;
- }
-
- private List<TblColRef> buildGroups() {
- List<TblColRef> groups = new ArrayList<TblColRef>();
-
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "string");
- TblColRef cf1 = new TblColRef(c1);
- groups.add(cf1);
-
- TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
- ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
- TblColRef cf2 = new TblColRef(c2);
- groups.add(cf2);
-
- return groups;
- }
-
- private List<FunctionDesc> buildAggregations() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
-
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- TblColRef priceCol = new TblColRef(ColumnDesc.mockup(t1, 7, "PRICE", "decimal(19,4)"));
- TblColRef sellerCol = new TblColRef(ColumnDesc.mockup(t1, 9, "SELLER_ID", "bigint"));
-
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("SUM");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue("PRICE");
- p1.setColRefs(ImmutableList.of(priceCol));
- f1.setParameter(p1);
- functions.add(f1);
-
- FunctionDesc f2 = new FunctionDesc();
- f2.setExpression("COUNT_DISTINCT");
- ParameterDesc p2 = new ParameterDesc();
- p2.setType("column");
- p2.setValue("SELLER_ID");
- p2.setColRefs(ImmutableList.of(sellerCol));
- f2.setParameter(p2);
- functions.add(f2);
-
- return functions;
- }
-
- private CompareTupleFilter buildFilter1(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
- ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter1);
- ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
- compareFilter.addChild(constantFilter1);
- return compareFilter;
- }
-
- private CompareTupleFilter buildFilter2(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
- ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter2);
- ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
- compareFilter.addChild(constantFilter2);
- return compareFilter;
- }
-
- @SuppressWarnings("unused")
- private TupleFilter buildAndFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
- CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
- LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
- andFilter.addChild(compareFilter1);
- andFilter.addChild(compareFilter2);
- return andFilter;
- }
-
- @SuppressWarnings("unused")
- private TupleFilter buildOrFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
- CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
- LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
- logicFilter.addChild(compareFilter1);
- logicFilter.addChild(compareFilter2);
- return logicFilter;
- }
}
[07/50] [abbrv] incubator-kylin git commit: KYLIN-749 add
STEAMING_TABLE def
Posted by sh...@apache.org.
KYLIN-749 add STEAMING_TABLE def
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/67a59370
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/67a59370
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/67a59370
Branch: refs/heads/streaming-localdict
Commit: 67a59370cbe342ddb9373045fb00d18dd7285206
Parents: 6ff8bef
Author: honma <ho...@ebay.com>
Authored: Mon May 4 15:21:21 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 14:38:58 2015 +0800
----------------------------------------------------------------------
.../invertedindex/test_streaming_table.json | 33 ++++++++
.../test_streaming_table_desc.json | 25 ++++++
.../test_kylin_left_join_model_desc.json | 87 +++++++++++---------
.../test_streaming_table_model_desc.json | 29 +++++++
.../localmeta/project/default.json | 5 ++
.../localmeta/table/DEFAULT.SREAMING_TABLE.json | 43 ++++++++++
6 files changed, 184 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
new file mode 100644
index 0000000..dc341aa
--- /dev/null
+++ b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
@@ -0,0 +1,33 @@
+{
+ "uuid": "daa53e80-41be-49a5-89ca-9fb729999812",
+ "name": "test_streaming_table",
+ "owner": null,
+ "version": null,
+ "cost": 10,
+ "status": "READY",
+ "segments": [
+ {
+ "uuid": null,
+ "name": "19700101000000_20140901000000",
+ "status": "READY",
+ "dictionaries": {
+ },
+ "storage_location_identifier": "",
+ "date_range_start": 0,
+ "date_range_end": 0,
+ "size_kb": 0,
+ "input_records": 0,
+ "input_records_size": 0,
+ "last_build_time": 0,
+ "last_build_job_id": null,
+ "create_time": null,
+ "binary_signature": null
+ }
+ ],
+ "last_modified": 0,
+ "descriptor": "test_streaming_table_desc",
+ "create_time": null,
+ "input_records_size": 0,
+ "size_kb": 0,
+ "source_records_count": 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
new file mode 100644
index 0000000..27cb053
--- /dev/null
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_desc.json
@@ -0,0 +1,25 @@
+{
+ "uuid": "74bf87b5-c7b5-4420-a12a-07f6b3864789",
+ "last_modified": 0,
+ "name": "test_streaming_table_desc",
+ "model_name": "test_streaming_table_model_desc",
+ "timestamp_dimension": "ts",
+ "value_dimensions": [
+ {
+ "table": "default.streaming_table",
+ "columns": [
+ "ts",
+ "minute_start",
+ "hour_start",
+ "itm",
+ "site"
+ ]
+ }
+ ],
+ "metrics": [
+ "gmv",
+ "item_count"
+ ],
+ "sharding": 1,
+ "slice_size": 1000
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index fa11c33..d05a08f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -1,35 +1,46 @@
{
- "uuid" : "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
- "name" : "test_kylin_left_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "left",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "left",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
+ "uuid": "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
+ "name": "test_kylin_left_join_model_desc",
+ "lookups": [
+ {
+ "table": "EDW.TEST_CAL_DT",
+ "join": {
+ "type": "left",
+ "primary_key": ["CAL_DT"],
+ "foreign_key": ["CAL_DT"]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "LEAF_CATEG_ID",
+ "SITE_ID"
+ ],
+ "foreign_key": [
+ "LEAF_CATEG_ID",
+ "LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "EDW.TEST_SITES",
+ "join": {
+ "type": "left",
+ "primary_key": ["SITE_ID"],
+ "foreign_key": ["LSTG_SITE_ID"]
+ }
+ },
+ {
+ "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "join": {
+ "type": "left",
+ "primary_key": ["SELLER_TYPE_CD"],
+ "foreign_key": ["SLR_SEGMENT_CD"]
+ }
}
- } ],
+ ],
"dimensions": [
{
"table": "default.test_kylin_fact",
@@ -89,13 +100,13 @@
"PRICE",
"ITEM_COUNT"
],
- "capacity" : "MEDIUM",
- "last_modified" : 1422435345352,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
+ "capacity": "MEDIUM",
+ "last_modified": 1422435345352,
+ "fact_table": "DEFAULT.TEST_KYLIN_FACT",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
new file mode 100644
index 0000000..66245ae
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
@@ -0,0 +1,29 @@
+{
+ "uuid": "ff527b94-f860-44c3-8452-93b177888732",
+ "name": "test_streaming_table_model_desc",
+ "dimensions": [
+ {
+ "table": "default.streaming_table",
+ "columns": [
+ "ts",
+ "minute_start",
+ "hour_start",
+ "itm",
+ "site"
+ ]
+ }
+ ],
+ "metrics": [
+ "gmv",
+ "item_count"
+ ],
+ "capacity": "MEDIUM",
+ "last_modified": 0,
+ "fact_table": "default.streaming_table",
+ "filter_condition": null,
+ "partition_desc": {
+ "partition_date_column": "default.streaming_table.ts",
+ "partition_date_start": 0,
+ "partition_type": "APPEND"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 55d8309..a11d4ca 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -33,6 +33,11 @@
"realization": "test_kylin_ii_inner_join"
},
{
+ "name": "test_streaming_table",
+ "type": "INVERTED_INDEX",
+ "realization": "test_streaming_table"
+ },
+ {
"name": "test_kylin_hybrid_left_join",
"type": "HYBRID",
"realization": "test_kylin_hybrid_left_join"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/67a59370/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
new file mode 100644
index 0000000..0e001c9
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
@@ -0,0 +1,43 @@
+{
+ "uuid": "e286e39e-40d7-44c2-8fa2-41b365123987",
+ "name": "STREAMING_TABLE",
+ "columns": [
+ {
+ "id": "1",
+ "name": "ts",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "2",
+ "name": "minute_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "3",
+ "name": "hour_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "4",
+ "name": "itm",
+ "datatype": "string"
+ },
+ {
+ "id": "5",
+ "name": "site",
+ "datatype": "string"
+ },
+ {
+ "id": "6",
+ "name": "gmv",
+ "datatype": "decimal(19,6)"
+ },
+ {
+ "id": "7",
+ "name": "item_count",
+ "datatype": "bigint"
+ }
+ ],
+ "database": "DEFAULT",
+ "last_modified": 0
+}
\ No newline at end of file
[08/50] [abbrv] incubator-kylin git commit: KYLIN-749 new test case
added, need manually insert data into kafka
Posted by sh...@apache.org.
KYLIN-749 new test case added, need manually insert data into kafka
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/09aba285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/09aba285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/09aba285
Branch: refs/heads/streaming-localdict
Commit: 09aba285da10fb1fe266d0ff1813c169bf570d80
Parents: 67a5937
Author: honma <ho...@ebay.com>
Authored: Tue May 5 16:41:47 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 16:41:47 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 6 +--
.../org/apache/kylin/common/util/JsonUtil.java | 9 ++--
.../org/apache/kylin/common/util/SortUtil.java | 21 ++++++++
.../invertedindex/test_streaming_table.json | 4 +-
.../test_streaming_table_model_desc.json | 1 +
.../streaming/test_streaming_table.json | 20 ++++++++
.../kylin/job/streaming/KafkaDataLoader.java | 46 +++++++++++++++++
.../java/org/apache/kylin/job/DataGenTest.java | 52 ++++++++++++++++++--
.../job/dataGen/StreamingDataGenerator.java | 36 ++++++++------
.../apache/kylin/metadata/model/ColumnDesc.java | 2 +-
.../kylin/streaming/JsonStreamParser.java | 8 +--
11 files changed, 169 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 7f12ac5..d184a24 100644
--- a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -15,7 +15,7 @@ public class DateFormat {
static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
- static SimpleDateFormat getDateFormat(String datePattern) {
+ public static SimpleDateFormat getDateFormat(String datePattern) {
ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
if (formatThreadLocal == null) {
threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
@@ -37,10 +37,6 @@ public class DateFormat {
return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis));
}
- public static String dateToString(Date date) {
- return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- }
-
public static String dateToString(Date date, String pattern) {
return getDateFormat(pattern).format(date);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 2a1e314..5b3a22c 100644
--- a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -18,12 +18,6 @@
package org.apache.kylin.common.util;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -32,12 +26,15 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import java.io.*;
+
public class JsonUtil {
// reuse the object mapper to save memory footprint
private static final ObjectMapper mapper = new ObjectMapper();
private static final ObjectMapper indentMapper = new ObjectMapper();
+
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
new file mode 100644
index 0000000..ac9f216
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
@@ -0,0 +1,21 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.base.Function;
+import com.google.common.collect.TreeMultimap;
+
+import java.util.Iterator;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class SortUtil {
+ public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
+ TreeMultimap<E, T> reorgnized = TreeMultimap.create();
+ while (input.hasNext()) {
+ T t = input.next();
+ E e = extractor.apply(t);
+ reorgnized.put(e, t);
+ }
+ return reorgnized.values().iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
index dc341aa..cbcca12 100644
--- a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
+++ b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
@@ -8,11 +8,11 @@
"segments": [
{
"uuid": null,
- "name": "19700101000000_20140901000000",
+ "name": "19700101000000_20190901000000",
"status": "READY",
"dictionaries": {
},
- "storage_location_identifier": "",
+ "storage_location_identifier": "KYLIN_2STEAMTEST",
"date_range_start": 0,
"date_range_end": 0,
"size_kb": 0,
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
index 66245ae..d0abb91 100644
--- a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
@@ -13,6 +13,7 @@
]
}
],
+ "lookups": [],
"metrics": [
"gmv",
"item_count"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/examples/test_case_data/localmeta/streaming/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/test_streaming_table.json b/examples/test_case_data/localmeta/streaming/test_streaming_table.json
new file mode 100644
index 0000000..537919a
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/test_streaming_table.json
@@ -0,0 +1,20 @@
+{
+ "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec909322",
+ "name": "test_streaming_table",
+ "zookeeper": "sandbox:2181",
+ "topic": "test_streaming_table",
+ "timeout": 60000,
+ "maxReadCount": 1000,
+ "bufferSize": 65536,
+ "iiName": "test_streaming_table",
+ "parserName": "org.apache.kylin.streaming.JsonStreamParser",
+ "partition": 1,
+ "last_modified": 0,
+ "brokers": [
+ {
+ "id": 0,
+ "host": "sandbox",
+ "port": 6667
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
new file mode 100644
index 0000000..5577dce
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
@@ -0,0 +1,46 @@
+package org.apache.kylin.job.streaming;
+
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+import kafka.producer.ProducerConfig;
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.streaming.KafkaConfig;
+import org.apache.kylin.streaming.StreamingManager;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class KafkaDataLoader {
+ /**
+ *
+ * @param args args[0] data file path, args[1] streaming name
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ StreamingManager streamingManager = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KafkaConfig kafkaConfig = streamingManager.getKafkaConfig(args[1]);
+
+ List<String> alldata = FileUtils.readLines(new File(args[0]));
+
+ Properties props = new Properties();
+ props.put("metadata.broker.list", "sandbox:6667");
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("request.required.acks", "1");
+
+ ProducerConfig config = new ProducerConfig(props);
+
+ Producer<String, String> producer = new Producer<String, String>(config);
+
+ for (int i = 0; i < alldata.size(); ++i) {
+ KeyedMessage<String, String> data = new KeyedMessage<String, String>(kafkaConfig.getTopic(), String.valueOf(i), alldata.get(i));
+ producer.send(data);
+ }
+ producer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index c05ddc7..4e3cf17 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -18,15 +18,28 @@
package org.apache.kylin.job;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
+import com.fasterxml.jackson.databind.type.SimpleType;
+import com.google.common.base.Function;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.SortUtil;
import org.apache.kylin.job.dataGen.FactTableGenerator;
+import org.apache.kylin.job.dataGen.StreamingDataGenerator;
+import org.apache.kylin.metadata.MetadataManager;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
/**
* Created by honma on 6/19/14.
@@ -56,6 +69,39 @@ public class DataGenTest extends LocalFileMetadataTestCase {
@Test
public void testStreaming() throws Exception {
+ int totalCount = 10000;
+ int counter = 0;
+
+ Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-01-06"), totalCount);
+
+ iterator = SortUtil.extractAndSort(iterator, new Function<String, Comparable>() {
+ public Comparable apply(String input) {
+ return getTsStr(input);
+ }
+ });
+
+ long lastTs = 0;
+ while (iterator.hasNext()) {
+ counter++;
+ String row = iterator.next();
+ System.out.println(row);
+ long ts = Long.parseLong(getTsStr(row));
+ Assert.assertTrue(ts >= lastTs);
+ lastTs = ts;
+ }
+ Assert.assertEquals(totalCount, counter);
}
+ final JavaType javaType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
+ final ObjectMapper objectMapper = new ObjectMapper();
+
+ private String getTsStr(String input) {
+ Map<String, String> json ;
+ try {
+ json = objectMapper.readValue(input.getBytes(), javaType);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return json.get("ts");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
index d822a9a..bf915e6 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
@@ -1,17 +1,18 @@
package org.apache.kylin.job.dataGen;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -25,23 +26,23 @@ public class StreamingDataGenerator {
private static Random random = new Random();
private static String[] decimalFormat = new String[] { "%.4f", "%.5f", "%.6f" };
- public static Iterator<List<String>> generate(final long start, final long end, final int count) {
+ public static Iterator<String> generate(final long start, final long end, final int count) {
final KylinConfig config = KylinConfig.getInstanceFromEnv();
final IIInstance ii = IIManager.getInstance(config).getII("test_streaming_table");
final IIDesc iiDesc = ii.getDescriptor();
- final MetadataManager metadataManager = MetadataManager.getInstance(config);
- final ColumnDesc[] columnDescs = metadataManager.getTableDesc(iiDesc.getFactTableName()).getColumns();
+ final List<TblColRef> columns = iiDesc.listAllColumns();
- return new Iterator<List<String>>() {
- private Map<String, String> values = Maps.newHashMap();
+ return new Iterator<String>() {
+ private Map<String, String> values = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ private int index = 0;
@Override
public boolean hasNext() {
- return false;
+ return this.index < count;
}
@Override
- public List<String> next() {
+ public String next() {
values.clear();
long ts = this.createTs(start, end);
values.put("ts", Long.toString(ts));
@@ -53,16 +54,19 @@ public class StreamingDataGenerator {
values.put("gmv", String.format(decimalFormat[random.nextInt(3)], random.nextFloat() * 100));
values.put("item_count", Integer.toString(random.nextInt(5)));
- if (values.size() != columnDescs.length) {
+ if (values.size() != columns.size()) {
throw new RuntimeException("the structure of streaming table has changed, need to modify generator too");
}
- List<String> ret = Lists.newArrayList();
- for (ColumnDesc columnDesc : columnDescs) {
- String name = columnDesc.getName();
- ret.add(values.get(name));
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ try {
+ JsonUtil.writeValue(os, values);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
}
- return ret;
+ index++;
+ return new String(os.toByteArray());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 8d7ebfd..203d304 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -73,7 +73,7 @@ public class ColumnDesc {
}
public void setDatatype(String datatype) {
- logger.info("setting datatype to " + datatype);
+ //logger.info("setting datatype to " + datatype);
this.datatype = datatype;
type = DataType.getInstance(datatype);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09aba285/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
index d762a14..d229647 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
@@ -34,6 +34,7 @@
package org.apache.kylin.streaming;
+import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.SimpleType;
@@ -54,18 +55,19 @@ import java.util.Map;
public final class JsonStreamParser implements StreamParser {
private static final Logger logger = LoggerFactory.getLogger(JsonStreamParser.class);
+ private static final JavaType javaType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
+ private static final ObjectMapper objectMapper = new ObjectMapper();
private final List<TblColRef> allColumns;
- public JsonStreamParser(List<TblColRef> allColumns){
+ public JsonStreamParser(List<TblColRef> allColumns) {
this.allColumns = allColumns;
}
@Override
public List<String> parse(StreamMessage streamMessage) {
try {
- Map<String, String> json = new ObjectMapper().readValue(
- streamMessage.getRawData(), MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class)));
+ Map<String, String> json = objectMapper.readValue(streamMessage.getRawData(), javaType);
ArrayList<String> result = Lists.newArrayList();
for (TblColRef column : allColumns) {
for (Map.Entry<String, String> entry : json.entrySet()) {
[39/50] [abbrv] incubator-kylin git commit: KYLIN-762 remove quartz
dependency
Posted by sh...@apache.org.
KYLIN-762 remove quartz dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/386e0c4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/386e0c4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/386e0c4e
Branch: refs/heads/streaming-localdict
Commit: 386e0c4e02893820b956c8e7ddad736bb04962af
Parents: e1e1aea
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed May 13 11:31:21 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed May 13 12:22:56 2015 +0800
----------------------------------------------------------------------
job/pom.xml | 9 ---------
pom.xml | 14 --------------
.../apache/kylin/rest/controller/CubeController.java | 1 -
3 files changed, 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/386e0c4e/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
index 2809323..c468cf9 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -97,15 +97,6 @@
</dependency>
<dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz</artifactId>
- </dependency>
- <dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz-jobs</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-daemon</groupId>
<artifactId>commons-daemon</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/386e0c4e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e750154..cb091a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,9 +91,6 @@
<!-- Metrics Codahale Version -->
<metrics.version>3.0.1</metrics.version>
- <!-- Quartz Version -->
- <quartz.version>2.2.1</quartz.version>
-
<!-- Curator.version Version -->
<curator.version>2.6.0</curator.version>
@@ -387,17 +384,6 @@
</dependency>
<dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz</artifactId>
- <version>${quartz.version}</version>
- </dependency>
- <dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz-jobs</artifactId>
- <version>${quartz.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/386e0c4e/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 78b4f79..556e21b 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -52,7 +52,6 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
[12/50] [abbrv] incubator-kylin git commit: KYLIN-749 remove
unnecessary system exit
Posted by sh...@apache.org.
KYLIN-749 remove unnecessary system exit
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/35f803c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/35f803c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/35f803c1
Branch: refs/heads/streaming-localdict
Commit: 35f803c1bfec9142a22ae354e5c999456b6ae3e4
Parents: 29da9ec
Author: honma <ho...@ebay.com>
Authored: Wed May 6 10:15:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:16:48 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/invertedindex/IICreateHTableJob.java | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/35f803c1/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 04313c9..77d0d55 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.job.tools.DeployCoprocessorCLI;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -33,12 +26,18 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.apache.kylin.job.tools.DeployCoprocessorCLI;
import org.apache.kylin.metadata.realization.IRealizationConstants;
/**
@@ -121,7 +120,6 @@ public class IICreateHTableJob extends AbstractHadoopJob {
public static void main(String[] args) throws Exception {
IICreateHTableJob job = new IICreateHTableJob();
job.setConf(HadoopUtil.newHBaseConfiguration(KylinConfig.getInstanceFromEnv().getStorageUrl()));
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
+ ToolRunner.run(job, args);
}
}
[11/50] [abbrv] incubator-kylin git commit: KYLIN-749 remove
unnecessary system exit
Posted by sh...@apache.org.
KYLIN-749 remove unnecessary system exit
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0578951a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0578951a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0578951a
Branch: refs/heads/streaming-localdict
Commit: 0578951a80bc92ebc0854d6d2be5b351e4a862b4
Parents: 59cb3c5
Author: honma <ho...@ebay.com>
Authored: Wed May 6 10:15:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:15:53 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/invertedindex/IICreateHTableJob.java | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0578951a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 04313c9..77d0d55 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.job.tools.DeployCoprocessorCLI;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -33,12 +26,18 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.apache.kylin.job.tools.DeployCoprocessorCLI;
import org.apache.kylin.metadata.realization.IRealizationConstants;
/**
@@ -121,7 +120,6 @@ public class IICreateHTableJob extends AbstractHadoopJob {
public static void main(String[] args) throws Exception {
IICreateHTableJob job = new IICreateHTableJob();
job.setConf(HadoopUtil.newHBaseConfiguration(KylinConfig.getInstanceFromEnv().getStorageUrl()));
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
+ ToolRunner.run(job, args);
}
}
[32/50] [abbrv] incubator-kylin git commit: Merge pull request #460
from qhzhou/new_igt_store
Posted by sh...@apache.org.
Merge pull request #460 from qhzhou/new_igt_store
KYLIN-728
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/73c275ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/73c275ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/73c275ed
Branch: refs/heads/streaming-localdict
Commit: 73c275ed36b55ee63935be591034026eda16628d
Parents: e008e2c 0d87e8f
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue May 12 14:01:31 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue May 12 14:01:31 2015 +0800
----------------------------------------------------------------------
.../kylin/common/util/BytesSerializer.java | 6 +-
.../org/apache/kylin/common/util/SSHClient.java | 5 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 14 +-
.../apache/kylin/dict/DictionarySerializer.java | 54 +++++
.../invertedindex/model/IIKeyValueCodec.java | 31 +--
.../job/hadoop/cubev2/InMemCubeBuilder.java | 197 +++++++++++++------
.../kylin/job/hadoop/cubev2/InMemCuboidJob.java | 2 +-
.../hadoop/cubev2/MapContextGTRecordWriter.java | 2 +-
.../metadata/serializer/DataTypeSerializer.java | 32 +--
.../kylin/storage/cube/CubeCodeSystem.java | 1 +
.../storage/cube/CubeHBaseReadonlyStore.java | 14 +-
.../kylin/storage/gridtable/GTBuilder.java | 1 -
.../kylin/storage/gridtable/GTComboStore.java | 112 +++++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 10 +-
.../storage/gridtable/GTInvertedIndex.java | 4 +-
.../kylin/storage/gridtable/GTRawScanner.java | 24 +--
.../kylin/storage/gridtable/GTRecord.java | 4 +-
.../kylin/storage/gridtable/GTRowBlock.java | 19 +-
.../storage/gridtable/GTSampleCodeSystem.java | 5 +-
.../kylin/storage/gridtable/IGTScanner.java | 6 +-
.../kylin/storage/gridtable/IGTStore.java | 18 +-
.../apache/kylin/storage/gridtable/ScanKey.java | 34 ++++
.../storage/gridtable/diskstore/FileSystem.java | 22 +++
.../gridtable/diskstore/GTDiskStore.java | 160 +++++++++++++++
.../gridtable/diskstore/HadoopFileSystem.java | 88 +++++++++
.../gridtable/diskstore/LocalFileSystem.java | 60 ++++++
.../gridtable/memstore/GTSimpleMemStore.java | 49 +++--
.../apache/kylin/storage/util/SizeOfUtil.java | 21 ++
.../invertedindex/IIStreamBuilder.java | 6 +-
29 files changed, 803 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
[10/50] [abbrv] incubator-kylin git commit: KYLIN-673 sampling part
data with a configurable percentage.
Posted by sh...@apache.org.
KYLIN-673 sampling part data with a configurable percentage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f2562639
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f2562639
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f2562639
Branch: refs/heads/streaming-localdict
Commit: f25626399b4764310fea7a8876add816c5010196
Parents: 6533a33
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed May 6 10:10:45 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed May 6 10:11:09 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 9 ++++++---
conf/kylin.properties | 4 +++-
.../test_case_data/sandbox/kylin.properties | 3 +++
.../kylin/job/constant/BatchConstants.java | 2 +-
.../apache/kylin/job/cube/CubingJobBuilder.java | 2 +-
.../kylin/job/hadoop/AbstractHadoopJob.java | 2 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 6 +++---
.../hadoop/cube/FactDistinctColumnsReducer.java | 21 +++++---------------
.../cube/FactDistinctHiveColumnsMapper.java | 19 +++++++-----------
9 files changed, 30 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index f1e5a73..7de6137 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -99,7 +99,7 @@ public class KylinConfig {
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
public static final String KYLIN_JOB_CUBING_IN_MEM = "kylin.job.cubing.inMem";
- public static final String KYLIN_JOB_CUBING_IN_MEM_SAMPLING_MAX = "kylin.job.cubing.inMem.sampling.max";
+ public static final String KYLIN_JOB_CUBING_IN_MEM_SAMPLING_PERCENT = "kylin.job.cubing.inMem.sampling.percent";
public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
@@ -483,8 +483,11 @@ public class KylinConfig {
return Boolean.parseBoolean(this.getOptional(KYLIN_JOB_CUBING_IN_MEM, "false"));
}
- public int getCubingInMemSamplingMax() {
- return Integer.parseInt(this.getOptional(KYLIN_JOB_CUBING_IN_MEM_SAMPLING_MAX, "100000"));
+ public int getCubingInMemSamplingPercent() {
+ int percent = Integer.parseInt(this.getOptional(KYLIN_JOB_CUBING_IN_MEM_SAMPLING_PERCENT, "5"));
+ percent = Math.max(percent, 1);
+ percent = Math.min(percent, 100);
+ return percent;
}
private String getOptional(String prop) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index 4932405..31e8356 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -38,7 +38,9 @@ kylin.job.yarn.app.rest.check.interval.seconds=10
# Whether calculate cube in mem in each mapper;
kylin.job.cubing.inMem=true
-kylin.job.cubing.inMem.sampling.max=100000
+
+#the percentage of the sampling, default 25%
+kylin.job.cubing.inMem.sampling.percent=25
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 5GB by default
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index acc843d..1830eaf 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -36,6 +36,9 @@ kylin.job.concurrent.max.limit=10
# Whether calculate cube in mem in each mapper;
kylin.job.cubing.inMem=false
+#the percentage of the sampling, default 25%
+kylin.job.cubing.inMem.sampling.percent=25
+
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 5GB by default
kylin.job.hbase.region.cut.small=5
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
index e5ca5af..b0444d0 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
@@ -50,7 +50,7 @@ public interface BatchConstants {
public static final String CFG_STATISTICS_ENABLED = "statistics.enabled";
public static final String CFG_STATISTICS_OUTPUT = "statistics.ouput";
- public static final String CFG_STATISTICS_SAMPLING_MAX = "statistics.sampling.max";
+ public static final String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
public static final String CFG_STATISTICS_CUBE_ESTIMATION = "cube_statistics.txt";
public static final String CFG_STATISTICS_CUBOID_ESTIMATION = "cuboid_statistics.seq";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index afca89a..27613df 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -287,7 +287,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
appendExecCmdParameters(cmd, "segmentname", seg.getName());
appendExecCmdParameters(cmd, "statisticsenabled", String.valueOf(inMemoryCubing()));
appendExecCmdParameters(cmd, "statisticsoutput", getStatisticsPath(seg, jobId));
- appendExecCmdParameters(cmd, "statisticssamplingmax", String.valueOf(engineConfig.getConfig().getCubingInMemSamplingMax()));
+ appendExecCmdParameters(cmd, "statisticssamplingpercent", String.valueOf(engineConfig.getConfig().getCubingInMemSamplingPercent()));
appendExecCmdParameters(cmd, "jobname", "Kylin_Fact_Distinct_Columns_" + seg.getCubeInstance().getName() + "_Step");
appendExecCmdParameters(cmd, "tablename", intermediateHiveTableName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index f24705c..1835954 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -88,7 +88,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_STATISTICS_ENABLED = OptionBuilder.withArgName("statisticsenabled").hasArg().isRequired(false).withDescription("Statistics enabled").create("statisticsenabled");
protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName("statisticsoutput").hasArg().isRequired(false).withDescription("Statistics output").create("statisticsoutput");
- protected static final Option OPTION_STATISTICS_SAMPLING_MAX = OptionBuilder.withArgName("statisticssamplingmax").hasArg().isRequired(false).withDescription("Statistics sampling max").create("statisticssamplingmax");
+ protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName("statisticssamplingpercent").hasArg().isRequired(false).withDescription("Statistics sampling percentage").create("statisticssamplingpercent");
protected String name;
protected String description;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index f7fe1ba..9e4b363 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -57,7 +57,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
options.addOption(OPTION_SEGMENT_NAME);
options.addOption(OPTION_STATISTICS_ENABLED);
options.addOption(OPTION_STATISTICS_OUTPUT);
- options.addOption(OPTION_STATISTICS_SAMPLING_MAX);
+ options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT);
parseOptions(options, args);
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
@@ -67,7 +67,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
String statistics_enabled = getOptionValue(OPTION_STATISTICS_ENABLED);
String statistics_output = getOptionValue(OPTION_STATISTICS_OUTPUT);
- String statistics_sampling_max = getOptionValue(OPTION_STATISTICS_SAMPLING_MAX);
+ String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT);
// ----------------------------------------------------------------------------
// add metadata to distributed cache
@@ -78,7 +78,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_ENABLED, statistics_enabled);
job.getConfiguration().set(BatchConstants.CFG_STATISTICS_OUTPUT, statistics_output);
- job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_MAX, statistics_sampling_max);
+ job.getConfiguration().set(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, statistics_sampling_percent);
log.info("Starting: " + job.getJobName());
setJobClasspath(job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index e1be38a..dd70117 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -58,7 +58,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
- private double averageSamplingRatio = 1;
+ private int SAMPING_PERCENTAGE = 5;
@Override
protected void setup(Context context) throws IOException {
@@ -80,6 +80,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
baseCuboidRowCountInMappers = Lists.newArrayList();
rowCountInCuboids = Maps.newHashMap();
cuboidHLLMap = Maps.newHashMap();
+ SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
}
}
@@ -112,7 +113,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
// for hll
long cuboidId = 0 - key.get();
- if (cuboidId <= baseCuboidId) {
for (Text value : values) {
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
ByteArray byteArray = new ByteArray(value.getBytes());
@@ -130,17 +130,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
cuboidHLLMap.put(cuboidId, hll);
}
}
- } else {
- int mapperCount = 0;
- averageSamplingRatio = 0;
- for (Text value : values) {
- averageSamplingRatio += Bytes.toDouble(value.getBytes());
- mapperCount++;
- }
-
- averageSamplingRatio = averageSamplingRatio / mapperCount;
-
- }
}
}
@@ -173,10 +162,10 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
msg = "Total cuboid number: \t" + allCuboids.size();
writeLine(out, msg);
- msg = "Avg samping ratio: \t" + averageSamplingRatio;
+ msg = "Samping percentage: \t" + SAMPING_PERCENTAGE;
writeLine(out, msg);
- writeLine(out, "The following statistics are collected based sampling data, not all data (only if samping ratio = 1).");
+ writeLine(out, "The following statistics are collected based sampling data.");
for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
if (baseCuboidRowCountInMappers.get(i) > 0) {
msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
@@ -223,7 +212,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
Collections.sort(allCuboids);
try {
for (long i : allCuboids) {
- writer.append(new LongWritable(i), new LongWritable((long) (rowCountInCuboids.get(i) / averageSamplingRatio)));
+ writer.append(new LongWritable(i), new LongWritable((long) (rowCountInCuboids.get(i) *100 / SAMPING_PERCENTAGE)));
}
} finally {
writer.close();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2562639/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
index d784cc1..d81bff1 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.kv.RowConstants;
@@ -58,7 +57,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private List<String> rowArray;
private HashFunction hf = null;
private int rowCount = 0;
- private int MAX_SAMPING_COUNT = 100000;
+ private int SAMPING_PERCENTAGE = 5;
@Override
protected void setup(Context context) throws IOException {
@@ -68,7 +67,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
rowArray = new ArrayList<String>(schema.getFields().size());
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
- MAX_SAMPING_COUNT = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_MAX, "100000"));
+ SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
cuboidScheduler = new CuboidScheduler(cubeDesc);
nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
@@ -128,18 +127,20 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
handleErrorRecord(record, ex);
}
- if (collectStatistics && rowCount < MAX_SAMPING_COUNT) {
+ if (collectStatistics && rowCount < SAMPING_PERCENTAGE) {
putRowKeyToHLL(rowArray);
}
- rowCount++;
+ if (rowCount++ == 100)
+ rowCount = 0;
}
private void putRowKeyToHLL(List<String> row) {
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
Hasher hc = hf.newHasher();
for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
- hc.putString(row.get(allCuboidsBitSet[i][position]));
+ if (row.get(allCuboidsBitSet[i][position]) != null)
+ hc.putString(row.get(allCuboidsBitSet[i][position]));
hc.putString(",");
}
@@ -161,12 +162,6 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
outputValue.set(hllBuf.array(), 0, hllBuf.position());
context.write(outputKey, outputValue);
}
-
- double samplingRatio = rowCount < MAX_SAMPING_COUNT ? 1.0 : ((double) MAX_SAMPING_COUNT) / rowCount;
- //output the total hll for this mapper;
- outputKey.set(0 - baseCuboidId - 1);
- outputValue.set(Bytes.toBytes(samplingRatio));
- context.write(outputKey, outputValue);
}
}
[49/50] [abbrv] incubator-kylin git commit: KYLIN-759 split
ICachableStorageEngine and IStorageEngine
Posted by sh...@apache.org.
KYLIN-759 split ICachableStorageEngine and IStorageEngine
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ac515a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ac515a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ac515a76
Branch: refs/heads/streaming-localdict
Commit: ac515a76e83b1da529b3e990f94867994983cc24
Parents: 0e42b2d
Author: honma <ho...@ebay.com>
Authored: Thu May 14 14:39:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 14 20:22:04 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/ICachableStorageEngine.java | 30 +++++++++++++++++++-
.../apache/kylin/storage/IStorageEngine.java | 21 --------------
.../kylin/storage/StorageEngineFactory.java | 6 ++--
.../AbstractCacheFledgedStorageEngine.java | 8 ++++--
.../cache/CacheFledgedDynamicStorageEngine.java | 18 +++---------
.../cache/CacheFledgedStaticStorageEngine.java | 22 +++++---------
.../kylin/storage/cube/CubeStorageEngine.java | 11 +++++--
.../kylin/storage/hbase/CubeStorageEngine.java | 12 ++++++--
.../hbase/InvertedIndexStorageEngine.java | 11 +++++--
.../storage/hybrid/HybridStorageEngine.java | 11 -------
.../kylin/storage/test/DynamicCacheTest.java | 9 ++++--
.../kylin/storage/test/StaticCacheTest.java | 9 ++++--
12 files changed, 90 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
index eadef09..e38350a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/ICachableStorageEngine.java
@@ -1,7 +1,35 @@
package org.apache.kylin.storage;
+import com.google.common.collect.Range;
+
/**
* Created by Hongbin Ma(Binmahone) on 5/14/15.
*/
-public interface ICachableStorageEngine extends IStorageEngine{
+public interface ICachableStorageEngine extends IStorageEngine {
+ /**
+ *
+ * being dynamic => getVolatilePeriod() return not null
+ * being dynamic => partition column of its realization not null
+ *
+ * @return true for static storage like cubes
+ * false for dynamic storage like II
+ */
+ boolean isDynamic();
+
+ /**
+ * volatile period is the period of time in which the returned data is not stable
+ * e.g. inverted index's last several minutes' data is dynamic as time goes by.
+ * data in this period cannot be cached
+ *
+ * This method should not be called before ITupleIterator.close() is called
+ *
+ * @return null if the underlying storage guarantees the data is static
+ */
+ Range<Long> getVolatilePeriod();
+
+ /**
+ * get the uuid for the realization assigned to this storage engine
+ * @return
+ */
+ String getStorageUUID();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
index d6b777a..46a94ca 100644
--- a/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/IStorageEngine.java
@@ -22,8 +22,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.tuple.TupleInfo;
-import com.google.common.collect.Range;
-
/**
*
* @author xjiang
@@ -33,25 +31,6 @@ public interface IStorageEngine {
ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo);
- /**
- *
- * being dynamic => getVolatilePeriod() return not null
- * being dynamic => partition column of its realization not null
- *
- * @return true for static storage like cubes
- * false for dynamic storage like II
- */
- boolean isDynamic();
- /**
- * volatile period is the period of time in which the returned data is not stable
- * e.g. inverted index's last several minutes' data is dynamic as time goes by.
- * data in this period cannot be cached
- *
- * This method should not be called before ITupleIterator.close() is called
- *
- * @return null if the underlying storage guarantees the data is static
- */
- Range<Long> getVolatilePeriod();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
index 29a53fd..0d720ab 100644
--- a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
+++ b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
@@ -45,14 +45,14 @@ public class StorageEngineFactory {
public static IStorageEngine getStorageEngine(IRealization realization) {
if (realization.getType() == RealizationType.INVERTED_INDEX) {
- IStorageEngine ret = new InvertedIndexStorageEngine((IIInstance) realization);
+ ICachableStorageEngine ret = new InvertedIndexStorageEngine((IIInstance) realization);
if (allowStorageLayerCache) {
return wrapWithCache(ret, realization);
} else {
return ret;
}
} else if (realization.getType() == RealizationType.CUBE) {
- IStorageEngine ret = new CubeStorageEngine((CubeInstance) realization);
+ ICachableStorageEngine ret = new CubeStorageEngine((CubeInstance) realization);
if (allowStorageLayerCache) {
return wrapWithCache(ret, realization);
} else {
@@ -63,7 +63,7 @@ public class StorageEngineFactory {
}
}
- private static IStorageEngine wrapWithCache(IStorageEngine underlyingStorageEngine, IRealization realization) {
+ private static IStorageEngine wrapWithCache(ICachableStorageEngine underlyingStorageEngine, IRealization realization) {
if (underlyingStorageEngine.isDynamic()) {
return new CacheFledgedDynamicStorageEngine(underlyingStorageEngine, getPartitionCol(realization));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
index d8e2fc0..09f0026 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedStorageEngine.java
@@ -8,6 +8,8 @@ import net.sf.ehcache.config.PersistenceConfiguration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.kylin.metadata.realization.StreamSQLDigest;
+import org.apache.kylin.metadata.tuple.TeeTupleItrListener;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.IStorageEngine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,15 +17,15 @@ import org.slf4j.LoggerFactory;
/**
* Created by Hongbin Ma(Binmahone) on 5/13/15.
*/
-public abstract class AbstractCacheFledgedStorageEngine {
+public abstract class AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFledgedStorageEngine.class);
protected static CacheManager cacheManager = CacheManager.create();
- protected final IStorageEngine underlyingStorage;
+ protected final ICachableStorageEngine underlyingStorage;
protected StreamSQLDigest streamSQLDigest;
protected boolean queryCacheExists;
- public AbstractCacheFledgedStorageEngine(IStorageEngine underlyingStorage) {
+ public AbstractCacheFledgedStorageEngine(ICachableStorageEngine underlyingStorage) {
this.underlyingStorage = underlyingStorage;
this.queryCacheExists = false;
this.makeCacheIfNecessary(underlyingStorage.getClass().getName());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
index 65672ae..0a4fd2a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicStorageEngine.java
@@ -12,7 +12,7 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.SQLDigestUtil;
import org.apache.kylin.metadata.realization.StreamSQLDigest;
import org.apache.kylin.metadata.tuple.*;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -24,14 +24,14 @@ import java.util.List;
/**
* Created by Hongbin Ma(Binmahone) on 5/11/15.
*/
-public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
+public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorageEngine {
private static final Logger logger = LoggerFactory.getLogger(CacheFledgedDynamicStorageEngine.class);
private final TblColRef partitionColRef;
private Range<Long> ts;
- public CacheFledgedDynamicStorageEngine(IStorageEngine underlyingStorage, TblColRef partitionColRef) {
+ public CacheFledgedDynamicStorageEngine(ICachableStorageEngine underlyingStorage, TblColRef partitionColRef) {
super(underlyingStorage);
this.partitionColRef = partitionColRef;
@@ -117,17 +117,7 @@ public class CacheFledgedDynamicStorageEngine extends AbstractCacheFledgedStorag
}
}
- @Override
- public Range<Long> getVolatilePeriod() {
- return underlyingStorage.getVolatilePeriod();
- }
-
- @Override
- public boolean isDynamic() {
- return true;
- }
-
- @Override
+ @Override
public void notify(List<ITuple> duplicated) {
Range<Long> cacheExclude = this.underlyingStorage.getVolatilePeriod();
if (cacheExclude != null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
index 860dea2..a309c24 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticStorageEngine.java
@@ -1,13 +1,15 @@
package org.apache.kylin.storage.cache;
-import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
import net.sf.ehcache.Cache;
import net.sf.ehcache.Element;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.StreamSQLDigest;
-import org.apache.kylin.metadata.tuple.*;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.metadata.tuple.TeeTupleIterator;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
@@ -18,10 +20,10 @@ import java.util.List;
/**
* Created by Hongbin Ma(Binmahone) on 5/11/15.
*/
-public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorageEngine implements IStorageEngine, TeeTupleItrListener {
+public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorageEngine {
private static final Logger logger = LoggerFactory.getLogger(CacheFledgedStaticStorageEngine.class);
- public CacheFledgedStaticStorageEngine(IStorageEngine underlyingStorage) {
+ public CacheFledgedStaticStorageEngine(ICachableStorageEngine underlyingStorage) {
super(underlyingStorage);
}
@@ -62,16 +64,6 @@ public class CacheFledgedStaticStorageEngine extends AbstractCacheFledgedStorage
}
@Override
- public Range<Long> getVolatilePeriod() {
- return underlyingStorage.getVolatilePeriod();
- }
-
- @Override
- public boolean isDynamic() {
- return false;
- }
-
- @Override
public void notify(List<ITuple> duplicated) {
StreamSQLResult newCacheEntry = new StreamSQLResult(duplicated, Ranges.<Long> all(), null);
cacheManager.getCache(this.underlyingStorage.getClass().getName()).put(new Element(streamSQLDigest, newCacheEntry));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
index a353f49..5fbcf98 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
@@ -21,7 +21,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.DerivedFilterTranslator;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-public class CubeStorageEngine implements IStorageEngine {
+public class CubeStorageEngine implements ICachableStorageEngine {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageEngine.class);
@@ -38,10 +38,12 @@ public class CubeStorageEngine implements IStorageEngine {
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
+ private final String uuid;
public CubeStorageEngine(CubeInstance cube) {
this.cubeInstance = cube;
this.cubeDesc = cube.getDescriptor();
+ this.uuid = cube.getUuid();
}
@Override
@@ -99,6 +101,11 @@ public class CubeStorageEngine implements IStorageEngine {
}
@Override
+ public String getStorageUUID() {
+ return this.uuid;
+ }
+
+ @Override
public boolean isDynamic() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index 699cc41..94c1dcc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -47,7 +47,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -59,7 +59,7 @@ import java.util.*;
/**
* @author xjiang, yangli9
*/
-public class CubeStorageEngine implements IStorageEngine {
+public class CubeStorageEngine implements ICachableStorageEngine {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageEngine.class);
@@ -68,10 +68,12 @@ public class CubeStorageEngine implements IStorageEngine {
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
+ private final String uuid;
public CubeStorageEngine(CubeInstance cube) {
this.cubeInstance = cube;
this.cubeDesc = cube.getDescriptor();
+ this.uuid = cube.getUuid();
}
@Override
@@ -138,6 +140,12 @@ public class CubeStorageEngine implements IStorageEngine {
}
@Override
+ public String getStorageUUID() {
+ return this.uuid;
+ }
+
+
+ @Override
public boolean isDynamic() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
index c94222e..de85190 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/InvertedIndexStorageEngine.java
@@ -25,7 +25,7 @@ import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.coprocessor.endpoint.EndpointTupleIterator;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -37,15 +37,17 @@ import java.util.ArrayList;
/**
* @author yangli9
*/
-public class InvertedIndexStorageEngine implements IStorageEngine {
+public class InvertedIndexStorageEngine implements ICachableStorageEngine {
private static Logger logger = LoggerFactory.getLogger(InvertedIndexStorageEngine.class);
private IISegment seg;
+ private String uuid;
private EndpointTupleIterator dataIterator;
public InvertedIndexStorageEngine(IIInstance ii) {
this.seg = ii.getFirstSegment();
+ this.uuid = ii.getUuid();
}
@Override
@@ -70,6 +72,11 @@ public class InvertedIndexStorageEngine implements IStorageEngine {
}
@Override
+ public String getStorageUUID() {
+ return this.uuid;
+ }
+
+ @Override
public boolean isDynamic() {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index f9cea05..b7219b5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -2,10 +2,8 @@ package org.apache.kylin.storage.hybrid;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.RangeUtil;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -64,15 +62,6 @@ public class HybridStorageEngine implements IStorageEngine {
return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
}
- @Override
- public Range<Long> getVolatilePeriod() {
- return RangeUtil.merge(historicalStorageEngine.getVolatilePeriod(), realtimeStorageEngine.getVolatilePeriod());
- }
-
- @Override
- public boolean isDynamic() {
- return true;
- }
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
index 0d962cb..3b587f1 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
@@ -12,7 +12,7 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.CacheFledgedDynamicStorageEngine;
import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
@@ -88,7 +88,7 @@ public class DynamicCacheTest {
final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
final List<Integer> returnedRowPerSearch = Lists.newArrayList();
- CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new IStorageEngine() {
+ CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new ICachableStorageEngine() {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter);
@@ -114,6 +114,11 @@ public class DynamicCacheTest {
public Range<Long> getVolatilePeriod() {
return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01"));
}
+
+ @Override
+ public String getStorageUUID() {
+ return "111ca32a-a33e-4b69-12aa-0bb8b1f8c191";
+ }
}, partitionCol);
sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ac515a76/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
index 9b69830..0133214 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
@@ -10,7 +10,7 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.ICachableStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.CacheFledgedStaticStorageEngine;
import org.apache.kylin.storage.tuple.Tuple;
@@ -45,7 +45,7 @@ public class StaticCacheTest {
final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
- CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new IStorageEngine() {
+ CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new ICachableStorageEngine() {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
underlyingSEHitCount.incrementAndGet();
@@ -61,6 +61,11 @@ public class StaticCacheTest {
public Range<Long> getVolatilePeriod() {
return null;
}
+
+ @Override
+ public String getStorageUUID() {
+ return "111ca32a-a33e-4b69-12aa-0bb8b1f8c092";
+ }
});
ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
[29/50] [abbrv] incubator-kylin git commit: KYLIN-749 clean up
Posted by sh...@apache.org.
KYLIN-749 clean up
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/876ac603
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/876ac603
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/876ac603
Branch: refs/heads/streaming-localdict
Commit: 876ac6032f86d9763a4ce645f0fab302dbea78e4
Parents: a2906ae
Author: honma <ho...@ebay.com>
Authored: Fri May 8 22:03:55 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri May 8 22:03:55 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/NumberDictionaryTest.java | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/876ac603/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 66b0a95..673d8b0 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -20,16 +20,6 @@ package org.apache.kylin.dict;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import static org.junit.Assert.*;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
@@ -75,7 +65,6 @@ public class NumberDictionaryTest {
checkCodec("-12345.123", "-9999999999987654.876;");
checkCodec("0", "00000000000000000");
checkCodec("0.0", "00000000000000000.0");
- checkCodec("123456789123456789", "-9999999999987654;");
}
private void checkCodec(String number, String code) {
[25/50] [abbrv] incubator-kylin git commit: KYLIN-753 Make the
dependency on hbase-common to "provided"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java b/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
index 49caeab..bd16246 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 6ac730e..bb1bab4 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -25,7 +25,7 @@ import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Writable;
public class BytesUtil {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index 3e8e765..76716b2 100644
--- a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java b/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
index 0233450..12b0c40 100644
--- a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
+++ b/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/Pair.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Pair.java b/common/src/main/java/org/apache/kylin/common/util/Pair.java
new file mode 100644
index 0000000..bd24401
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/Pair.java
@@ -0,0 +1,132 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.Serializable;
+
+
+/**
+ * A generic class for pairs. Copied from org.apache.hadoop.hbase.util.Pair
+ * @param <T1>
+ * @param <T2>
+ */
+public class Pair<T1, T2> implements Serializable
+{
+ private static final long serialVersionUID = -3986244606585552569L;
+ protected T1 first = null;
+ protected T2 second = null;
+
+ /**
+ * Default constructor.
+ */
+ public Pair()
+ {
+ }
+
+ /**
+ * Constructor
+ * @param a operand
+ * @param b operand
+ */
+ public Pair(T1 a, T2 b)
+ {
+ this.first = a;
+ this.second = b;
+ }
+
+ /**
+ * Constructs a new pair, inferring the type via the passed arguments
+ * @param <T1> type for first
+ * @param <T2> type for second
+ * @param a first element
+ * @param b second element
+ * @return a new pair containing the passed arguments
+ */
+ public static <T1,T2> Pair<T1,T2> newPair(T1 a, T2 b) {
+ return new Pair<T1,T2>(a, b);
+ }
+
+ /**
+ * Replace the first element of the pair.
+ * @param a operand
+ */
+ public void setFirst(T1 a)
+ {
+ this.first = a;
+ }
+
+ /**
+ * Replace the second element of the pair.
+ * @param b operand
+ */
+ public void setSecond(T2 b)
+ {
+ this.second = b;
+ }
+
+ /**
+ * Return the first element stored in the pair.
+ * @return T1
+ */
+ public T1 getFirst()
+ {
+ return first;
+ }
+
+ /**
+ * Return the second element stored in the pair.
+ * @return T2
+ */
+ public T2 getSecond()
+ {
+ return second;
+ }
+
+ private static boolean equals(Object x, Object y)
+ {
+ return (x == null && y == null) || (x != null && x.equals(y));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public boolean equals(Object other)
+ {
+ return other instanceof Pair && equals(first, ((Pair)other).first) &&
+ equals(second, ((Pair)other).second);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ if (first == null)
+ return (second == null) ? 0 : second.hashCode() + 1;
+ else if (second == null)
+ return first.hashCode() + 2;
+ else
+ return first.hashCode() * 17 + second.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "{" + getFirst() + "," + getSecond() + "}";
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index ee7de59..9c2d559 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -26,7 +26,7 @@ import java.util.HashSet;
import java.util.Random;
import java.util.Set;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 75f984c..581c769 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -22,7 +22,7 @@ import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 9caa29c..48db0d8 100644
--- a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -19,7 +19,7 @@
package org.apache.kylin.cube.common;
import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 6d57c6d..c3cf716 100644
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index adc07b2..a2add5c 100644
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -21,7 +21,7 @@ package org.apache.kylin.cube.kv;
import java.util.Arrays;
import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index 1113667..5b6cd91 100644
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
index 99c2485..2ab12b9 100644
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
+++ b/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
@@ -19,7 +19,7 @@
package org.apache.kylin.cube.kv;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
index f671978..aefec85 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
@@ -4,7 +4,7 @@ import static org.junit.Assert.*;
import java.io.IOException;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.junit.After;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 5e0ea2b..534ee20 100644
--- a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index a789910..c8164e4 100644
--- a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.*;
import java.util.Arrays;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 467456a..7b0b877 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.dict.lookup.ReadableTable;
import org.apache.kylin.dict.lookup.TableReader;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 787d50c..107f209 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -18,7 +18,7 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 5857171..d8e5476 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -18,7 +18,7 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java b/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
index ddf2d6d..0bec6a1 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
@@ -18,7 +18,7 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
public class StringBytesConverter implements BytesConverter<String> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 5c4b56d..8b8a815 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -28,7 +28,7 @@ import java.lang.ref.SoftReference;
import java.util.Arrays;
import java.util.HashMap;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
index 665f090..1226d72 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
index 6e5f7d9..ee71e78 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index fab95f1..50d4cbc 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Sets;
import org.apache.kylin.common.util.Array;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
index dd0f0d6..86fa635 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict;
import java.io.File;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dict.lookup.FileTable;
import org.apache.kylin.dict.lookup.LookupBytesTable;
import org.junit.After;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 405fece..66b0a95 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -20,7 +20,17 @@ package org.apache.kylin.dict;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.hadoop.hbase.util.Bytes;
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 1985868..138ae33 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -22,7 +22,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
index 7bd2266..2b65ca6 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import com.ning.compress.lzf.LZFDecoder;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index c300fc5..55104b5 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -22,7 +22,7 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index 4efa16b..9bd9234 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -23,7 +23,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.*;
import org.apache.commons.net.util.Base64;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
index e6adb09..2809323 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -138,8 +138,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
- <!-- This is default scope to be included in the job jar, we depends on
- hbase utils like Bytes, ImmutableBytesWritable etc. -->
+ <scope>provided</scope>
</dependency>
<!-- Env & Test -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 6acb05b..47a3145 100644
--- a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Maps;
import org.apache.kylin.common.util.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index e4a0fae..a01f462 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -21,7 +21,7 @@ package org.apache.kylin.job.cube;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.job.AbstractJobBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
index b9c0d60..948f25e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
@@ -24,7 +24,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hive.hcatalog.data.HCatRecord;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
index eb23d19..91fdc71 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperBase.java
@@ -1,7 +1,7 @@
package org.apache.kylin.job.hadoop.cube;
import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.mr.KylinMapper;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
index 70b30f6..1391f15 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
@@ -23,7 +23,7 @@ import java.util.HashSet;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.util.ByteArray;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index dd70117..c671da9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
index d81bff1..0b75084 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
@@ -22,7 +22,7 @@ import com.google.common.collect.Lists;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyValueCreator.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyValueCreator.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyValueCreator.java
index a8d06de..0318a8b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyValueCreator.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyValueCreator.java
@@ -1,7 +1,7 @@
package org.apache.kylin.job.hadoop.cube;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index b4a93a1..f5ee5b9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -27,8 +27,8 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.hadoop.io.Text;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index 9a02564..29cdc9a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -35,8 +35,8 @@ package org.apache.kylin.job.hadoop.cubev2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
index cb723f5..41237d7 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
@@ -3,7 +3,7 @@ package org.apache.kylin.job.hadoop.cubev2;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.kylin.cube.CubeSegment;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
index 07d16f8..b49af1a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
index cc71351..7c673da 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index 6a06f35..69d241f 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
index 3b6036c..fc086cb 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
@@ -21,7 +21,7 @@ package org.apache.kylin.job.hadoop.invertedindex;
import java.io.IOException;
import java.util.HashSet;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
index da8f5d9..6406ad4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
@@ -20,7 +20,7 @@ package org.apache.kylin.job.hadoop.invertedindex;
import java.io.IOException;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.hcatalog.data.HCatRecord;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
index 5088404..d7b2735 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java b/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
index fc52260..a39b273 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java b/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
index a40f265..4e84bf3 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.persistence.HBaseConnection;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java b/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
index 8479391..8dcfa6d 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HbaseStreamingInput.java
@@ -12,7 +12,7 @@ import java.util.concurrent.Semaphore;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9acd2c3..329eef1 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -26,7 +26,7 @@ import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
index bc7d8e3..5262936 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
@@ -27,7 +27,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.junit.After;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
index f906fcb..563633f 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
@@ -25,7 +25,7 @@ import java.math.BigDecimal;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.types.Pair;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
index 0baa738..4e362b3 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
@@ -25,7 +25,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
index a92da92..1dc5c44 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
/**
* Created by hongbin on 5/15/14.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
index 034c90e..2c01fb6 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
@@ -27,7 +27,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
index d4f202d..671d249 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/BigDecimalSerializer.java
@@ -22,7 +22,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/metadata/src/main/java/org/apache/kylin/metadata/serializer/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DateTimeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DateTimeSerializer.java
index 8223b69..f817693 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DateTimeSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DateTimeSerializer.java
@@ -2,7 +2,7 @@ package org.apache.kylin.metadata.serializer;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.common.util.DateFormat;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
index 8e6f1fe..0ebeb78 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DoubleSerializer.java
@@ -20,7 +20,7 @@ package org.apache.kylin.metadata.serializer;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.kylin.metadata.model.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
index 0be9e5a..ba9ff8b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/LongSerializer.java
@@ -20,7 +20,7 @@ package org.apache.kylin.metadata.serializer;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
index 4b0b2cd..8686fb7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/StringSerializer.java
@@ -2,7 +2,7 @@ package org.apache.kylin.metadata.serializer;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/server/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/AclService.java b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
index 12de99c..8413f47 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 05031b9..000ba60 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -23,7 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 62a9567..027202d 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.metrics.QueryMetrics;
import org.apache.kylin.rest.model.ColumnMeta;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/server/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/UserService.java b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index 416db1f..b5d366a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.rest.security.UserManager;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.userdetails.User;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
index d17954b..70567e5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
@@ -6,7 +6,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowConstants;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index 0074a05..f721148 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -12,8 +12,8 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.cube.CubeSegment;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
index 47882ca..1633f25 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeStorageEngine.java
@@ -6,7 +6,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 2aac09b..a9b4f92 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -19,12 +19,9 @@
package org.apache.kylin.storage.hbase;
import java.text.MessageFormat;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Set;
+import java.util.*;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
@@ -35,7 +32,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.cube.CubeSegment;
@@ -216,9 +213,9 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
}
private void applyFuzzyFilter(Scan scan, HBaseKeyRange keyRange) {
- List<Pair<byte[], byte[]>> fuzzyKeys = keyRange.getFuzzyKeys();
+ List<org.apache.kylin.common.util.Pair<byte[], byte[]>> fuzzyKeys = keyRange.getFuzzyKeys();
if (fuzzyKeys != null && fuzzyKeys.size() > 0) {
- FuzzyRowFilter rowFilter = new FuzzyRowFilter(fuzzyKeys);
+ FuzzyRowFilter rowFilter = new FuzzyRowFilter(convertToHBasePair(fuzzyKeys));
Filter filter = scan.getFilter();
if (filter != null) {
@@ -233,6 +230,16 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
}
}
+ private List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> convertToHBasePair(List<org.apache.kylin.common.util.Pair<byte[], byte[]>> pairList) {
+ List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> result = Lists.newArrayList();
+ for(org.apache.kylin.common.util.Pair pair : pairList) {
+ org.apache.hadoop.hbase.util.Pair element = new org.apache.hadoop.hbase.util.Pair(pair.getFirst(), pair.getSecond());
+ result.add(element);
+ }
+
+ return result;
+ }
+
private void closeScanner() {
if (logger.isDebugEnabled() && scan != null) {
logger.debug("Scan " + scan.toString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index 920527e..ab75a3c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -28,8 +28,8 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.storage.tuple.TupleInfo;
import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index 743769f..ed24958 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -21,7 +21,7 @@ package org.apache.kylin.storage.hbase;
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.kv.RowKeyColumnOrder;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index c7be838..e683fbc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -24,8 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/PingHBaseCLI.java b/storage/src/main/java/org/apache/kylin/storage/hbase/PingHBaseCLI.java
index f844cdb..1d4ec68 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/PingHBaseCLI.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/PingHBaseCLI.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.util.HadoopUtil;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
index 139853e..73bfd25 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
@@ -2,7 +2,7 @@ package org.apache.kylin.storage.hbase.coprocessor;
import com.google.common.collect.Sets;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java
index f905d32..0610ea5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregators.java
@@ -27,7 +27,7 @@ import java.util.List;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
index 93ade44..5613ecc 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
@@ -21,7 +21,7 @@ package org.apache.kylin.storage.hbase.coprocessor;
import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.junit.Test;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
index 8202ae8..b6ee99a 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/RowAggregatorsTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/RowAggregatorsTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/RowAggregatorsTest.java
index 7dacc2e..9458014 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/RowAggregatorsTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/RowAggregatorsTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.*;
import java.util.Arrays;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.junit.Test;
/**
[47/50] [abbrv] incubator-kylin git commit: KYLIN-759 clean log
setttings
Posted by sh...@apache.org.
KYLIN-759 clean log setttings
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/60d94fa5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/60d94fa5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/60d94fa5
Branch: refs/heads/streaming-localdict
Commit: 60d94fa55bfed1876dbbecb821aa5323c5cbffb4
Parents: 9179235
Author: honma <ho...@ebay.com>
Authored: Thu May 14 20:17:41 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 14 20:22:04 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/cache/StreamSQLResult.java | 1 -
.../apache/kylin/storage/cache/EhcacheTest.java | 10 +++-
streaming/src/test/resources/log4j.xml | 51 --------------------
3 files changed, 9 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60d94fa5/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 924044b..57dbd67 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -17,7 +17,6 @@ import java.util.NavigableMap;
/**
*/
-
public class StreamSQLResult {
private Range<Long> timeCovered;
private NavigableMap<Long, List<ITuple>> sortedRows;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60d94fa5/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
index fe390de..2751196 100644
--- a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -1,5 +1,6 @@
package org.apache.kylin.storage.cache;
+import com.google.common.collect.Lists;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
@@ -7,9 +8,10 @@ import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.Configuration;
import net.sf.ehcache.config.PersistenceConfiguration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
-
import org.junit.Test;
+import java.util.List;
+
/**
*/
public class EhcacheTest {
@@ -36,6 +38,12 @@ public class EhcacheTest {
System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
byte[] blob = new byte[(1024 * 40 * 1024)];//400M
+ List<String> manyObjects = Lists.newArrayList();
+ for (int i = 0; i < 10000; i++) {
+ manyObjects.add(new String("" + i));
+ }
+ testCache.put(new Element("0", manyObjects));
+
testCache.put(new Element("1", blob));
System.out.println(testCache.get("1") == null);
System.out.println(testCache.getSize());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60d94fa5/streaming/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/streaming/src/test/resources/log4j.xml b/streaming/src/test/resources/log4j.xml
deleted file mode 100644
index a0afa96..0000000
--- a/streaming/src/test/resources/log4j.xml
+++ /dev/null
@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- ~ /*
- ~
- ~ * Licensed to the Apache Software Foundation (ASF) under one or more
- ~
- ~ * contributor license agreements. See the NOTICE file distributed with
- ~
- ~ * this work for additional information regarding copyright ownership.
- ~
- ~ * The ASF licenses this file to You under the Apache License, Version 2.0
- ~
- ~ * (the "License"); you may not use this file except in compliance with
- ~
- ~ * the License. You may obtain a copy of the License at
- ~
- ~ *
- ~
- ~ * http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ *
- ~
- ~ * Unless required by applicable law or agreed to in writing, software
- ~
- ~ * distributed under the License is distributed on an "AS IS" BASIS,
- ~
- ~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~
- ~ * See the License for the specific language governing permissions and
- ~
- ~ * limitations under the License.
- ~
- ~ */
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration debug="true"
- xmlns:log4j='http://jakarta.apache.org/log4j/'>
-
- <appender name="consoleAppender" class="org.apache.log4j.ConsoleAppender">
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} - %m%n"/>
- </layout>
- </appender>
-
- <root>
- <level value="INFO"/>
- <appender-ref ref="consoleAppender"/>
- </root>
-
-</log4j:configuration>
\ No newline at end of file
[50/50] [abbrv] incubator-kylin git commit: KYLIN-702 make default
block size to 64M for hive flat table.
Posted by sh...@apache.org.
KYLIN-702 make default block size to 64M for hive flat table.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4cf65f07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4cf65f07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4cf65f07
Branch: refs/heads/streaming-localdict
Commit: 4cf65f070bc9ddb05bba1414fcc67b1a991e64c7
Parents: 60d94fa
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri May 15 10:36:57 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri May 15 10:37:14 2015 +0800
----------------------------------------------------------------------
conf/kylin_hive_conf.xml | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4cf65f07/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/conf/kylin_hive_conf.xml b/conf/kylin_hive_conf.xml
index d990175..8bd2c10 100644
--- a/conf/kylin_hive_conf.xml
+++ b/conf/kylin_hive_conf.xml
@@ -7,6 +7,12 @@
</property>
<property>
+ <name>dfs.block.size</name>
+ <value>67108864</value>
+ <description>Block size</description>
+ </property>
+
+ <property>
<name>hive.exec.compress.output</name>
<value>true</value>
<description>enable compress</description>
@@ -47,8 +53,13 @@
<description>Enable hive file merge on map-reduce job</description>
</property>
<property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+ <property>
<name>hive.merge.size.per.task</name>
- <value>64000000</value>
+ <value>256000000</value>
<description>Size for the merged file</description>
</property>
</configuration>
\ No newline at end of file
[06/50] [abbrv] incubator-kylin git commit: KYLIN-749 add stream data
generator
Posted by sh...@apache.org.
KYLIN-749 add stream data generator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6ff8befc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6ff8befc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6ff8befc
Branch: refs/heads/streaming-localdict
Commit: 6ff8befcf8eb1a1897f6550651c837c5463a9598
Parents: 6533a33
Author: honma <ho...@ebay.com>
Authored: Tue May 5 14:38:31 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 5 14:38:31 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/TimeUtil.java | 17 +++++
.../apache/kylin/common/util/TimeUtilTest.java | 44 +++++++++++
.../kylin/job/streaming/StreamingBootstrap.java | 36 +++++----
.../java/org/apache/kylin/job/DataGenTest.java | 4 +
.../job/dataGen/StreamingDataGenerator.java | 78 ++++++++++++++++++++
5 files changed, 160 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ff8befc/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
new file mode 100644
index 0000000..01fc2c1
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -0,0 +1,17 @@
+package org.apache.kylin.common.util;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/4/15.
+ */
+public class TimeUtil {
+ private static int ONE_MINUTE_TS = 60 * 1000;
+ private static int ONE_HOUR_TS = 60 * 60 * 1000;
+
+ public static long getMinuteStart(long ts) {
+ return ts / ONE_MINUTE_TS * ONE_MINUTE_TS;
+ }
+
+ public static long getHourStart(long ts) {
+ return ts / ONE_HOUR_TS * ONE_HOUR_TS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ff8befc/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
new file mode 100644
index 0000000..1442412
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -0,0 +1,44 @@
+package org.apache.kylin.common.util;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/4/15.
+ */
+public class TimeUtilTest {
+ public static long normalizeTime(long timeMillis, NormalizeUnit unit) {
+ Calendar a = Calendar.getInstance();
+ Calendar b = Calendar.getInstance();
+ b.clear();
+
+ a.setTimeInMillis(timeMillis);
+ if (unit == NormalizeUnit.MINUTE) {
+ b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE));
+ } else if (unit == NormalizeUnit.HOUR) {
+ b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0);
+ }
+ return b.getTimeInMillis();
+ }
+
+ @Test
+ public void basicTest() throws ParseException {
+ java.text.DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+
+ long t1 = dateFormat.parse("2012/01/01 00:00:01").getTime();
+ Assert.assertEquals(normalizeTime(t1, NormalizeUnit.HOUR), TimeUtil.getHourStart(t1));
+ Assert.assertEquals(normalizeTime(t1, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t1));
+
+ long t2 = dateFormat.parse("2012/12/31 11:02:01").getTime();
+ Assert.assertEquals(normalizeTime(t2, NormalizeUnit.HOUR), TimeUtil.getHourStart(t2));
+ Assert.assertEquals(normalizeTime(t2, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t2));
+ }
+
+ public enum NormalizeUnit {
+ MINUTE, HOUR
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ff8befc/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 3e8f3f4..1688fc2 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -34,27 +34,30 @@
package org.apache.kylin.job.streaming;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
import kafka.api.OffsetRequest;
import kafka.cluster.Broker;
import kafka.javaapi.PartitionMetadata;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.streaming.*;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Constructor;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
/**
* Created by qianzhou on 3/26/15.
@@ -69,6 +72,12 @@ public class StreamingBootstrap {
private Map<String, KafkaConsumer> kafkaConsumers = Maps.newConcurrentMap();
+ private StreamingBootstrap(KylinConfig kylinConfig) {
+ this.kylinConfig = kylinConfig;
+ this.streamingManager = StreamingManager.getInstance(kylinConfig);
+ this.iiManager = IIManager.getInstance(kylinConfig);
+ }
+
public static StreamingBootstrap getInstance(KylinConfig kylinConfig) {
final StreamingBootstrap bootstrap = new StreamingBootstrap(kylinConfig);
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@@ -80,12 +89,6 @@ public class StreamingBootstrap {
return bootstrap;
}
- private StreamingBootstrap(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- this.streamingManager = StreamingManager.getInstance(kylinConfig);
- this.iiManager = IIManager.getInstance(kylinConfig);
- }
-
private static Broker getLeadBroker(KafkaConfig kafkaConfig, int partitionId) {
final PartitionMetadata partitionMetadata = KafkaRequester.getPartitionMetadata(kafkaConfig.getTopic(), partitionId, kafkaConfig.getBrokers(), kafkaConfig);
if (partitionMetadata != null && partitionMetadata.errorCode() == 0) {
@@ -137,11 +140,7 @@ public class StreamingBootstrap {
streamingOffset = Math.max(streamingOffset, earliestOffset);
logger.info("starting offset is " + streamingOffset);
- if (!HBaseConnection.tableExists(kylinConfig.getStorageUrl(), iiSegment.getStorageLocationIdentifier())) {
- logger.error("no htable:" + iiSegment.getStorageLocationIdentifier() + " found");
- throw new IllegalStateException("please create htable:" + iiSegment.getStorageLocationIdentifier() + " first");
- }
-
+ IICreateHTableJob.main(new String[] { "-iiname", "nous_ii", "-htablename", "KYLIN_2SKJ8JNOUS" });
KafkaConsumer consumer = new KafkaConsumer(kafkaConfig.getTopic(), partitionId, streamingOffset, kafkaConfig.getBrokers(), kafkaConfig, parallelism);
kafkaConsumers.put(getKey(streaming, partitionId), consumer);
@@ -166,7 +165,6 @@ public class StreamingBootstrap {
}
}
-
}
private String getKey(String streaming, int partitionId) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ff8befc/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index 89e964e..c05ddc7 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -54,4 +54,8 @@ public class DataGenTest extends LocalFileMetadataTestCase {
DeployUtil.overrideFactTableData(content, "default.test_kylin_fact");
}
+ @Test
+ public void testStreaming() throws Exception {
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ff8befc/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
new file mode 100644
index 0000000..d822a9a
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
@@ -0,0 +1,78 @@
+package org.apache.kylin.job.dataGen;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.TimeUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class StreamingDataGenerator {
+ private static final Logger logger = LoggerFactory.getLogger(StreamingDataGenerator.class);
+ private static Random random = new Random();
+ private static String[] decimalFormat = new String[] { "%.4f", "%.5f", "%.6f" };
+
+ public static Iterator<List<String>> generate(final long start, final long end, final int count) {
+ final KylinConfig config = KylinConfig.getInstanceFromEnv();
+ final IIInstance ii = IIManager.getInstance(config).getII("test_streaming_table");
+ final IIDesc iiDesc = ii.getDescriptor();
+ final MetadataManager metadataManager = MetadataManager.getInstance(config);
+ final ColumnDesc[] columnDescs = metadataManager.getTableDesc(iiDesc.getFactTableName()).getColumns();
+
+ return new Iterator<List<String>>() {
+ private Map<String, String> values = Maps.newHashMap();
+
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public List<String> next() {
+ values.clear();
+ long ts = this.createTs(start, end);
+ values.put("ts", Long.toString(ts));
+ values.put("minute_start", Long.toString(TimeUtil.getMinuteStart(ts)));
+ values.put("hour_start", Long.toString(TimeUtil.getHourStart(ts)));
+ values.put("itm", Integer.toString(random.nextInt(20)));
+ values.put("site", Integer.toString(random.nextInt(5)));
+
+ values.put("gmv", String.format(decimalFormat[random.nextInt(3)], random.nextFloat() * 100));
+ values.put("item_count", Integer.toString(random.nextInt(5)));
+
+ if (values.size() != columnDescs.length) {
+ throw new RuntimeException("the structure of streaming table has changed, need to modify generator too");
+ }
+
+ List<String> ret = Lists.newArrayList();
+ for (ColumnDesc columnDesc : columnDescs) {
+ String name = columnDesc.getName();
+ ret.add(values.get(name));
+ }
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ }
+
+ private long createTs(final long start, final long end) {
+ return start + (long) (random.nextDouble() * (end - start));
+ }
+ };
+ }
+
+}
[26/50] [abbrv] incubator-kylin git commit: KYLIN-753 Make the
dependency on hbase-common to "provided"
Posted by sh...@apache.org.
KYLIN-753 Make the dependency on hbase-common to "provided"
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8f65c1c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8f65c1c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8f65c1c6
Branch: refs/heads/streaming-localdict
Commit: 8f65c1c6e249e134dda2bc4c1619b55de569f7bb
Parents: 39ada70
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu May 7 16:46:07 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu May 7 17:27:31 2015 +0800
----------------------------------------------------------------------
.../common/persistence/HBaseResourceStore.java | 2 +-
.../kylin/common/restclient/RestClient.java | 2 +-
.../org/apache/kylin/common/util/ByteArray.java | 2 +-
.../org/apache/kylin/common/util/Bytes.java | 2260 ++++++++++++++++++
.../apache/kylin/common/util/BytesSplitter.java | 2 +-
.../org/apache/kylin/common/util/BytesUtil.java | 2 +-
.../kylin/common/util/CliCommandExecutor.java | 2 +-
.../common/util/HBaseRegionSizeCalculator.java | 2 +-
.../java/org/apache/kylin/common/util/Pair.java | 132 +
.../common/util/HyperLogLogCounterTest.java | 2 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../kylin/cube/common/RowKeySplitter.java | 2 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 2 +-
.../apache/kylin/cube/kv/RowKeyColumnIO.java | 2 +-
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 2 +-
.../apache/kylin/cube/kv/RowValueDecoder.java | 2 +-
.../org/apache/kylin/cube/CubeSegmentsTest.java | 2 +-
.../apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +-
.../apache/kylin/cube/kv/RowKeyEncoderTest.java | 2 +-
.../apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../org/apache/kylin/dict/NumberDictionary.java | 2 +-
.../kylin/dict/NumberDictionaryBuilder.java | 2 +-
.../apache/kylin/dict/StringBytesConverter.java | 2 +-
.../org/apache/kylin/dict/TrieDictionary.java | 2 +-
.../kylin/dict/lookup/FileTableReader.java | 2 +-
.../kylin/dict/lookup/LookupBytesTable.java | 2 +-
.../apache/kylin/dict/lookup/LookupTable.java | 2 +-
.../org/apache/kylin/dict/LookupTableTest.java | 2 +-
.../apache/kylin/dict/NumberDictionaryTest.java | 12 +-
.../kylin/dict/lookup/LookupTableTest.java | 2 +-
.../index/CompressedValueContainer.java | 2 +-
.../invertedindex/index/RawTableRecord.java | 2 +-
.../kylin/invertedindex/model/IIDesc.java | 2 +-
job/pom.xml | 3 +-
.../kylin/job/common/ShellExecutable.java | 2 +-
.../apache/kylin/job/cube/CubingJobBuilder.java | 2 +-
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../job/hadoop/cube/BaseCuboidMapperBase.java | 2 +-
.../cube/FactDistinctColumnsCombiner.java | 2 +-
.../hadoop/cube/FactDistinctColumnsReducer.java | 2 +-
.../cube/FactDistinctHiveColumnsMapper.java | 2 +-
.../kylin/job/hadoop/cube/KeyValueCreator.java | 2 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 4 +-
.../job/hadoop/cubev2/InMemCubeBuilder.java | 4 +-
.../hadoop/cubev2/MapContextGTRecordWriter.java | 2 +-
.../hadoop/cubev2/MergeCuboidFromHBaseJob.java | 2 +-
.../cubev2/MergeCuboidFromHBaseMapper.java | 2 +-
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 2 +-
.../IIDistinctColumnsCombiner.java | 2 +-
.../invertedindex/IIDistinctColumnsMapper.java | 2 +-
.../invertedindex/IIDistinctColumnsReducer.java | 2 +-
.../kylin/job/tools/DeployCoprocessorCLI.java | 2 +-
.../job/tools/GridTableHBaseBenchmark.java | 4 +-
.../kylin/job/tools/HbaseStreamingInput.java | 2 +-
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../job/hadoop/cube/CubeHFileMapper2Test.java | 2 +-
.../hadoop/cube/HiveToBaseCuboidMapperTest.java | 2 +-
.../job/hadoop/cube/NDCuboidMapperTest.java | 2 +-
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
.../job/tools/ColumnCardinalityReducerTest.java | 2 +-
.../serializer/BigDecimalSerializer.java | 2 +-
.../metadata/serializer/DateTimeSerializer.java | 2 +-
.../metadata/serializer/DoubleSerializer.java | 2 +-
.../metadata/serializer/LongSerializer.java | 2 +-
.../metadata/serializer/StringSerializer.java | 2 +-
.../apache/kylin/rest/service/AclService.java | 2 +-
.../apache/kylin/rest/service/JobService.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 2 +-
.../apache/kylin/rest/service/UserService.java | 2 +-
.../kylin/storage/cube/CubeCodeSystem.java | 2 +-
.../storage/cube/CubeHBaseReadonlyStore.java | 4 +-
.../kylin/storage/cube/CubeStorageEngine.java | 2 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 23 +-
.../kylin/storage/hbase/CubeStorageEngine.java | 4 +-
.../storage/hbase/DerivedFilterTranslator.java | 2 +-
.../kylin/storage/hbase/HBaseKeyRange.java | 4 +-
.../kylin/storage/hbase/PingHBaseCLI.java | 2 +-
.../hbase/coprocessor/FilterDecorator.java | 2 +-
.../observer/ObserverAggregators.java | 2 +-
.../hbase/coprocessor/RowProjectorTest.java | 2 +-
.../observer/AggregateRegionObserverTest.java | 2 +-
.../observer/RowAggregatorsTest.java | 2 +-
82 files changed, 2502 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
index 59903c5..2868368 100644
--- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index b81218d..64a8a74 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -30,7 +30,7 @@ import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index b7362f6..e63f904 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -21,7 +21,7 @@ package org.apache.kylin.common.util;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8f65c1c6/common/src/main/java/org/apache/kylin/common/util/Bytes.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Bytes.java b/common/src/main/java/org/apache/kylin/common/util/Bytes.java
new file mode 100644
index 0000000..3bbc93c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/Bytes.java
@@ -0,0 +1,2260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+import sun.misc.Unsafe;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.Charset;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.SecureRandom;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.*;
+
+/**
+ * Utility class that handles byte arrays, conversions to/from other types,
+ * comparisons, hash code generation, manufacturing keys for HashMaps or
+ * HashSets, etc.
+ *
+ * Copied from org.apache.hadoop.hbase.util.Bytes
+ */
+public class Bytes {
+ //HConstants.UTF8_ENCODING should be updated if this changed
+ /**
+ * When we encode strings, we always specify UTF8 encoding
+ */
+ private static final String UTF8_ENCODING = "UTF-8";
+
+ //HConstants.UTF8_CHARSET should be updated if this changed
+ /**
+ * When we encode strings, we always specify UTF8 encoding
+ */
+ private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
+
+ //HConstants.EMPTY_BYTE_ARRAY should be updated if this changed
+ private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+
+ private static final Log LOG = LogFactory.getLog(Bytes.class);
+
+ /**
+ * Size of boolean in bytes
+ */
+ public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
+
+ /**
+ * Size of byte in bytes
+ */
+ public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
+
+ /**
+ * Size of char in bytes
+ */
+ public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
+
+ /**
+ * Size of double in bytes
+ */
+ public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
+
+ /**
+ * Size of float in bytes
+ */
+ public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
+
+ /**
+ * Size of int in bytes
+ */
+ public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
+
+ /**
+ * Size of long in bytes
+ */
+ public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
+
+ /**
+ * Size of short in bytes
+ */
+ public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
+
+
+ /**
+ * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
+ * Estimate based on study of jhat and jprofiler numbers.
+ */
+ // JHat says BU is 56 bytes.
+ // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
+ public static final int ESTIMATED_HEAP_TAX = 16;
+
+
+ /**
+ * Returns length of the byte array, returning 0 if the array is null.
+ * Useful for calculating sizes.
+ *
+ * @param b byte array, which can be null
+ * @return 0 if b is null, otherwise returns length
+ */
+ final public static int len(byte[] b) {
+ return b == null ? 0 : b.length;
+ }
+
+ /**
+ * Byte array comparator class.
+ */
+ public static class ByteArrayComparator implements RawComparator<byte[]> {
+ /**
+ * Constructor
+ */
+ public ByteArrayComparator() {
+ super();
+ }
+
+ @Override
+ public int compare(byte[] left, byte[] right) {
+ return compareTo(left, right);
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(b1, s1, l1, b2, s2, l2);
+ }
+ }
+
+ /**
+ * A {@link ByteArrayComparator} that treats the empty array as the largest value.
+ * This is useful for comparing row end keys for regions.
+ */
+ // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region
+ // boundaries. Thus semantically, we should treat empty byte array as the smallest value
+ // while comparing row keys, start keys etc; but as the largest value for comparing
+ // region boundaries for endKeys.
+ public static class RowEndKeyComparator extends ByteArrayComparator {
+ @Override
+ public int compare(byte[] left, byte[] right) {
+ return compare(left, 0, left.length, right, 0, right.length);
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ if (b1 == b2 && s1 == s2 && l1 == l2) {
+ return 0;
+ }
+ if (l1 == 0) {
+ return l2; //0 or positive
+ }
+ if (l2 == 0) {
+ return -1;
+ }
+ return super.compare(b1, s1, l1, b2, s2, l2);
+ }
+ }
+
+ /**
+ * Pass this to TreeMaps where byte [] are keys.
+ */
+ public final static Comparator<byte[]> BYTES_COMPARATOR = new ByteArrayComparator();
+
+ /**
+ * Use comparing byte arrays, byte-by-byte
+ */
+ public final static RawComparator<byte[]> BYTES_RAWCOMPARATOR = new ByteArrayComparator();
+
+ /**
+ * Read byte-array written with a WritableableUtils.vint prefix.
+ *
+ * @param in Input to read from.
+ * @return byte array read off <code>in</code>
+ * @throws java.io.IOException e
+ */
+ public static byte[] readByteArray(final DataInput in)
+ throws IOException {
+ int len = WritableUtils.readVInt(in);
+ if (len < 0) {
+ throw new NegativeArraySizeException(Integer.toString(len));
+ }
+ byte[] result = new byte[len];
+ in.readFully(result, 0, len);
+ return result;
+ }
+
+ /**
+ * Read byte-array written with a WritableableUtils.vint prefix.
+ * IOException is converted to a RuntimeException.
+ *
+ * @param in Input to read from.
+ * @return byte array read off <code>in</code>
+ */
+ public static byte[] readByteArrayThrowsRuntime(final DataInput in) {
+ try {
+ return readByteArray(in);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Write byte-array with a WritableableUtils.vint prefix.
+ *
+ * @param out output stream to be written to
+ * @param b array to write
+ * @throws IOException e
+ */
+ public static void writeByteArray(final DataOutput out, final byte[] b)
+ throws IOException {
+ if (b == null) {
+ WritableUtils.writeVInt(out, 0);
+ } else {
+ writeByteArray(out, b, 0, b.length);
+ }
+ }
+
+ /**
+ * Write byte-array to out with a vint length prefix.
+ *
+ * @param out output stream
+ * @param b array
+ * @param offset offset into array
+ * @param length length past offset
+ * @throws IOException e
+ */
+ public static void writeByteArray(final DataOutput out, final byte[] b,
+ final int offset, final int length)
+ throws IOException {
+ WritableUtils.writeVInt(out, length);
+ out.write(b, offset, length);
+ }
+
+ /**
+ * Write byte-array from src to tgt with a vint length prefix.
+ *
+ * @param tgt target array
+ * @param tgtOffset offset into target array
+ * @param src source array
+ * @param srcOffset source offset
+ * @param srcLength source length
+ * @return New offset in src array.
+ */
+ public static int writeByteArray(final byte[] tgt, final int tgtOffset,
+ final byte[] src, final int srcOffset, final int srcLength) {
+ byte[] vint = vintToBytes(srcLength);
+ System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
+ int offset = tgtOffset + vint.length;
+ System.arraycopy(src, srcOffset, tgt, offset, srcLength);
+ return offset + srcLength;
+ }
+
+ /**
+ * Put bytes at the specified byte array position.
+ *
+ * @param tgtBytes the byte array
+ * @param tgtOffset position in the array
+ * @param srcBytes array to write out
+ * @param srcOffset source offset
+ * @param srcLength source length
+ * @return incremented offset
+ */
+ public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
+ int srcOffset, int srcLength) {
+ System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
+ return tgtOffset + srcLength;
+ }
+
+ /**
+ * Write a single byte out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param b byte to write out
+ * @return incremented offset
+ */
+ public static int putByte(byte[] bytes, int offset, byte b) {
+ bytes[offset] = b;
+ return offset + 1;
+ }
+
+ /**
+ * Add the whole content of the ByteBuffer to the bytes arrays. The ByteBuffer is modified.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param buf ByteBuffer to write out
+ * @return incremented offset
+ */
+ public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
+ int len = buf.remaining();
+ buf.get(bytes, offset, len);
+ return offset + len;
+ }
+
+ /**
+ * Returns a new byte array, copied from the given {@code buf},
+ * from the index 0 (inclusive) to the limit (exclusive),
+ * regardless of the current position.
+ * The position and the other index parameters are not changed.
+ *
+ * @param buf a byte buffer
+ * @return the byte array
+ * @see #getBytes(ByteBuffer)
+ */
+ public static byte[] toBytes(ByteBuffer buf) {
+ ByteBuffer dup = buf.duplicate();
+ dup.position(0);
+ return readBytes(dup);
+ }
+
+ private static byte[] readBytes(ByteBuffer buf) {
+ byte[] result = new byte[buf.remaining()];
+ buf.get(result);
+ return result;
+ }
+
+ /**
+ * @param b Presumed UTF-8 encoded byte array.
+ * @return String made from <code>b</code>
+ */
+ public static String toString(final byte[] b) {
+ if (b == null) {
+ return null;
+ }
+ return toString(b, 0, b.length);
+ }
+
+ /**
+ * Joins two byte arrays together using a separator.
+ *
+ * @param b1 The first byte array.
+ * @param sep The separator to use.
+ * @param b2 The second byte array.
+ */
+ public static String toString(final byte[] b1,
+ String sep,
+ final byte[] b2) {
+ return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
+ }
+
+ /**
+ * This method will convert utf8 encoded bytes into a string. If
+ * the given byte array is null, this method will return null.
+ *
+ * @param b Presumed UTF-8 encoded byte array.
+ * @param off offset into array
+ * @param len length of utf-8 sequence
+ * @return String made from <code>b</code> or null
+ */
+ public static String toString(final byte[] b, int off, int len) {
+ if (b == null) {
+ return null;
+ }
+ if (len == 0) {
+ return "";
+ }
+ return new String(b, off, len, UTF8_CHARSET);
+ }
+
+ /**
+ * Write a printable representation of a byte array.
+ *
+ * @param b byte array
+ * @return string
+ * @see #toStringBinary(byte[], int, int)
+ */
+ public static String toStringBinary(final byte[] b) {
+ if (b == null)
+ return "null";
+ return toStringBinary(b, 0, b.length);
+ }
+
+ /**
+ * Converts the given byte buffer to a printable representation,
+ * from the index 0 (inclusive) to the limit (exclusive),
+ * regardless of the current position.
+ * The position and the other index parameters are not changed.
+ *
+ * @param buf a byte buffer
+ * @return a string representation of the buffer's binary contents
+ * @see #toBytes(ByteBuffer)
+ * @see #getBytes(ByteBuffer)
+ */
+ public static String toStringBinary(ByteBuffer buf) {
+ if (buf == null)
+ return "null";
+ if (buf.hasArray()) {
+ return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
+ }
+ return toStringBinary(toBytes(buf));
+ }
+
+ /**
+ * Write a printable representation of a byte array. Non-printable
+ * characters are hex escaped in the format \\x%02X, eg:
+ * \x00 \x05 etc
+ *
+ * @param b array to write out
+ * @param off offset to start at
+ * @param len length to write
+ * @return string output
+ */
+ public static String toStringBinary(final byte[] b, int off, int len) {
+ StringBuilder result = new StringBuilder();
+ // Just in case we are passed a 'len' that is > buffer length...
+ if (off >= b.length) return result.toString();
+ if (off + len > b.length) len = b.length - off;
+ for (int i = off; i < off + len; ++i) {
+ int ch = b[i] & 0xFF;
+ if ((ch >= '0' && ch <= '9')
+ || (ch >= 'A' && ch <= 'Z')
+ || (ch >= 'a' && ch <= 'z')
+ || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
+ result.append((char) ch);
+ } else {
+ result.append(String.format("\\x%02X", ch));
+ }
+ }
+ return result.toString();
+ }
+
+ private static boolean isHexDigit(char c) {
+ return
+ (c >= 'A' && c <= 'F') ||
+ (c >= '0' && c <= '9');
+ }
+
+ /**
+ * Takes a ASCII digit in the range A-F0-9 and returns
+ * the corresponding integer/ordinal value.
+ *
+ * @param ch The hex digit.
+ * @return The converted hex value as a byte.
+ */
+ public static byte toBinaryFromHex(byte ch) {
+ if (ch >= 'A' && ch <= 'F')
+ return (byte) ((byte) 10 + (byte) (ch - 'A'));
+ // else
+ return (byte) (ch - '0');
+ }
+
+ public static byte[] toBytesBinary(String in) {
+ // this may be bigger than we need, but let's be safe.
+ byte[] b = new byte[in.length()];
+ int size = 0;
+ for (int i = 0; i < in.length(); ++i) {
+ char ch = in.charAt(i);
+ if (ch == '\\' && in.length() > i + 1 && in.charAt(i + 1) == 'x') {
+ // ok, take next 2 hex digits.
+ char hd1 = in.charAt(i + 2);
+ char hd2 = in.charAt(i + 3);
+
+ // they need to be A-F0-9:
+ if (!isHexDigit(hd1) ||
+ !isHexDigit(hd2)) {
+ // bogus escape code, ignore:
+ continue;
+ }
+ // turn hex ASCII digit -> number
+ byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
+
+ b[size++] = d;
+ i += 3; // skip 3
+ } else {
+ b[size++] = (byte) ch;
+ }
+ }
+ // resize:
+ byte[] b2 = new byte[size];
+ System.arraycopy(b, 0, b2, 0, size);
+ return b2;
+ }
+
+ /**
+ * Converts a string to a UTF-8 byte array.
+ *
+ * @param s string
+ * @return the byte array
+ */
+ public static byte[] toBytes(String s) {
+ return s.getBytes(UTF8_CHARSET);
+ }
+
+ /**
+ * Convert a boolean to a byte array. True becomes -1
+ * and false becomes 0.
+ *
+ * @param b value
+ * @return <code>b</code> encoded in a byte array.
+ */
+ public static byte[] toBytes(final boolean b) {
+ return new byte[]{b ? (byte) -1 : (byte) 0};
+ }
+
+ /**
+ * Reverses {@link #toBytes(boolean)}
+ *
+ * @param b array
+ * @return True or false.
+ */
+ public static boolean toBoolean(final byte[] b) {
+ if (b.length != 1) {
+ throw new IllegalArgumentException("Array has wrong size: " + b.length);
+ }
+ return b[0] != (byte) 0;
+ }
+
+ /**
+ * Convert a long value to a byte array using big-endian.
+ *
+ * @param val value to convert
+ * @return the byte array
+ */
+ public static byte[] toBytes(long val) {
+ byte[] b = new byte[8];
+ for (int i = 7; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * Converts a byte array to a long value. Reverses
+ * {@link #toBytes(long)}
+ *
+ * @param bytes array
+ * @return the long value
+ */
+ public static long toLong(byte[] bytes) {
+ return toLong(bytes, 0, SIZEOF_LONG);
+ }
+
+ /**
+ * Converts a byte array to a long value. Assumes there will be
+ * {@link #SIZEOF_LONG} bytes available.
+ *
+ * @param bytes bytes
+ * @param offset offset
+ * @return the long value
+ */
+ public static long toLong(byte[] bytes, int offset) {
+ return toLong(bytes, offset, SIZEOF_LONG);
+ }
+
+ /**
+ * Converts a byte array to a long value.
+ *
+ * @param bytes array of bytes
+ * @param offset offset into array
+ * @param length length of data (must be {@link #SIZEOF_LONG})
+ * @return the long value
+ * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
+ * if there's not enough room in the array at the offset indicated.
+ */
+ public static long toLong(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_LONG || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return toLongUnsafe(bytes, offset);
+ } else {
+ long l = 0;
+ for (int i = offset; i < offset + length; i++) {
+ l <<= 8;
+ l ^= bytes[i] & 0xFF;
+ }
+ return l;
+ }
+ }
+
+ private static IllegalArgumentException
+ explainWrongLengthOrOffset(final byte[] bytes,
+ final int offset,
+ final int length,
+ final int expectedLength) {
+ String reason;
+ if (length != expectedLength) {
+ reason = "Wrong length: " + length + ", expected " + expectedLength;
+ } else {
+ reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+ + " capacity of the array: " + bytes.length;
+ }
+ return new IllegalArgumentException(reason);
+ }
+
+ /**
+ * Put a long value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val long to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putLong(byte[] bytes, int offset, long val) {
+ if (bytes.length - offset < SIZEOF_LONG) {
+ throw new IllegalArgumentException("Not enough room to put a long at"
+ + " offset " + offset + " in a " + bytes.length + " byte array");
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return putLongUnsafe(bytes, offset, val);
+ } else {
+ for (int i = offset + 7; i > offset; i--) {
+ bytes[i] = (byte) val;
+ val >>>= 8;
+ }
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_LONG;
+ }
+ }
+
+ /**
+ * Put a long value out to the specified byte array position (Unsafe).
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val long to write out
+ * @return incremented offset
+ */
+ public static int putLongUnsafe(byte[] bytes, int offset, long val) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ val = Long.reverseBytes(val);
+ }
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putLong(bytes, (long) offset +
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
+ return offset + SIZEOF_LONG;
+ }
+
+ /**
+ * Presumes float encoded as IEEE 754 floating-point "single format"
+ *
+ * @param bytes byte array
+ * @return Float made from passed byte array.
+ */
+ public static float toFloat(byte[] bytes) {
+ return toFloat(bytes, 0);
+ }
+
+ /**
+ * Presumes float encoded as IEEE 754 floating-point "single format"
+ *
+ * @param bytes array to convert
+ * @param offset offset into array
+ * @return Float made from passed byte array.
+ */
+ public static float toFloat(byte[] bytes, int offset) {
+ return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+ }
+
+ /**
+ * @param bytes byte array
+ * @param offset offset to write to
+ * @param f float value
+ * @return New offset in <code>bytes</code>
+ */
+ public static int putFloat(byte[] bytes, int offset, float f) {
+ return putInt(bytes, offset, Float.floatToRawIntBits(f));
+ }
+
+ /**
+ * @param f float value
+ * @return the float represented as byte []
+ */
+ public static byte[] toBytes(final float f) {
+ // Encode it as int
+ return Bytes.toBytes(Float.floatToRawIntBits(f));
+ }
+
+ /**
+ * @param bytes byte array
+ * @return Return double made from passed bytes.
+ */
+ public static double toDouble(final byte[] bytes) {
+ return toDouble(bytes, 0);
+ }
+
+ /**
+ * @param bytes byte array
+ * @param offset offset where double is
+ * @return Return double made from passed bytes.
+ */
+ public static double toDouble(final byte[] bytes, final int offset) {
+ return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+ }
+
+ /**
+ * @param bytes byte array
+ * @param offset offset to write to
+ * @param d value
+ * @return New offset into array <code>bytes</code>
+ */
+ public static int putDouble(byte[] bytes, int offset, double d) {
+ return putLong(bytes, offset, Double.doubleToLongBits(d));
+ }
+
+ /**
+ * Serialize a double as the IEEE 754 double format output. The resultant
+ * array will be 8 bytes long.
+ *
+ * @param d value
+ * @return the double represented as byte []
+ */
+ public static byte[] toBytes(final double d) {
+ // Encode it as a long
+ return Bytes.toBytes(Double.doubleToRawLongBits(d));
+ }
+
+ /**
+ * Convert an int value to a byte array. Big-endian. Same as what DataOutputStream.writeInt
+ * does.
+ *
+ * @param val value
+ * @return the byte array
+ */
+ public static byte[] toBytes(int val) {
+ byte[] b = new byte[4];
+ for (int i = 3; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * Converts a byte array to an int value
+ *
+ * @param bytes byte array
+ * @return the int value
+ */
+ public static int toInt(byte[] bytes) {
+ return toInt(bytes, 0, SIZEOF_INT);
+ }
+
+ /**
+ * Converts a byte array to an int value
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @return the int value
+ */
+ public static int toInt(byte[] bytes, int offset) {
+ return toInt(bytes, offset, SIZEOF_INT);
+ }
+
+ /**
+ * Converts a byte array to an int value
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @param length length of int (has to be {@link #SIZEOF_INT})
+ * @return the int value
+ * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
+ * if there's not enough room in the array at the offset indicated.
+ */
+ public static int toInt(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_INT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return toIntUnsafe(bytes, offset);
+ } else {
+ int n = 0;
+ for (int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+ }
+
+ /**
+ * Converts a byte array to an int value (Unsafe version)
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @return the int value
+ */
+ public static int toIntUnsafe(byte[] bytes, int offset) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ return Integer.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getInt(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+ } else {
+ return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getInt(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
+ }
+ }
+
+ /**
+ * Converts a byte array to an short value (Unsafe version)
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @return the short value
+ */
+ public static short toShortUnsafe(byte[] bytes, int offset) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ return Short.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getShort(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+ } else {
+ return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getShort(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
+ }
+ }
+
+ /**
+ * Converts a byte array to an long value (Unsafe version)
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @return the long value
+ */
+ public static long toLongUnsafe(byte[] bytes, int offset) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ return Long.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getLong(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
+ } else {
+ return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getLong(bytes,
+ (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
+ }
+ }
+
+ /**
+ * Converts a byte array to an int value
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @param length how many bytes should be considered for creating int
+ * @return the int value
+ * @throws IllegalArgumentException if there's not enough room in the array at the offset
+ * indicated.
+ */
+ public static int readAsInt(byte[] bytes, int offset, final int length) {
+ if (offset + length > bytes.length) {
+ throw new IllegalArgumentException("offset (" + offset + ") + length (" + length
+ + ") exceed the" + " capacity of the array: " + bytes.length);
+ }
+ int n = 0;
+ for (int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+
+ /**
+ * Put an int value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val int to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putInt(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_INT) {
+ throw new IllegalArgumentException("Not enough room to put an int at"
+ + " offset " + offset + " in a " + bytes.length + " byte array");
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return putIntUnsafe(bytes, offset, val);
+ } else {
+ for (int i = offset + 3; i > offset; i--) {
+ bytes[i] = (byte) val;
+ val >>>= 8;
+ }
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_INT;
+ }
+ }
+
+ /**
+ * Put an int value out to the specified byte array position (Unsafe).
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val int to write out
+ * @return incremented offset
+ */
+ public static int putIntUnsafe(byte[] bytes, int offset, int val) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ val = Integer.reverseBytes(val);
+ }
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putInt(bytes, (long) offset +
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
+ return offset + SIZEOF_INT;
+ }
+
+ /**
+ * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
+ *
+ * @param val value
+ * @return the byte array
+ */
+ public static byte[] toBytes(short val) {
+ byte[] b = new byte[SIZEOF_SHORT];
+ b[1] = (byte) val;
+ val >>= 8;
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * Converts a byte array to a short value
+ *
+ * @param bytes byte array
+ * @return the short value
+ */
+ public static short toShort(byte[] bytes) {
+ return toShort(bytes, 0, SIZEOF_SHORT);
+ }
+
+ /**
+ * Converts a byte array to a short value
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @return the short value
+ */
+ public static short toShort(byte[] bytes, int offset) {
+ return toShort(bytes, offset, SIZEOF_SHORT);
+ }
+
+ /**
+ * Converts a byte array to a short value
+ *
+ * @param bytes byte array
+ * @param offset offset into array
+ * @param length length, has to be {@link #SIZEOF_SHORT}
+ * @return the short value
+ * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
+ * or if there's not enough room in the array at the offset indicated.
+ */
+ public static short toShort(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_SHORT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return toShortUnsafe(bytes, offset);
+ } else {
+ short n = 0;
+ n ^= bytes[offset] & 0xFF;
+ n <<= 8;
+ n ^= bytes[offset + 1] & 0xFF;
+ return n;
+ }
+ }
+
+ /**
+ * Returns a new byte array, copied from the given {@code buf},
+ * from the position (inclusive) to the limit (exclusive).
+ * The position and the other index parameters are not changed.
+ *
+ * @param buf a byte buffer
+ * @return the byte array
+ * @see #toBytes(ByteBuffer)
+ */
+ public static byte[] getBytes(ByteBuffer buf) {
+ return readBytes(buf.duplicate());
+ }
+
+ /**
+ * Put a short value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val short to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putShort(byte[] bytes, int offset, short val) {
+ if (bytes.length - offset < SIZEOF_SHORT) {
+ throw new IllegalArgumentException("Not enough room to put a short at"
+ + " offset " + offset + " in a " + bytes.length + " byte array");
+ }
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
+ return putShortUnsafe(bytes, offset, val);
+ } else {
+ bytes[offset + 1] = (byte) val;
+ val >>= 8;
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_SHORT;
+ }
+ }
+
+ /**
+ * Put a short value out to the specified byte array position (Unsafe).
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val short to write out
+ * @return incremented offset
+ */
+ public static int putShortUnsafe(byte[] bytes, int offset, short val) {
+ if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
+ val = Short.reverseBytes(val);
+ }
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putShort(bytes, (long) offset +
+ org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
+ return offset + SIZEOF_SHORT;
+ }
+
+ /**
+ * Put an int value as short out to the specified byte array position. Only the lower 2 bytes of
+ * the short will be put into the array. The caller of the API need to make sure they will not
+ * loose the value by doing so. This is useful to store an unsigned short which is represented as
+ * int in other parts.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val value to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putAsShort(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_SHORT) {
+ throw new IllegalArgumentException("Not enough room to put a short at"
+ + " offset " + offset + " in a " + bytes.length + " byte array");
+ }
+ bytes[offset + 1] = (byte) val;
+ val >>= 8;
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_SHORT;
+ }
+
+ /**
+ * Convert a BigDecimal value to a byte array
+ *
+ * @param val
+ * @return the byte array
+ */
+ public static byte[] toBytes(BigDecimal val) {
+ byte[] valueBytes = val.unscaledValue().toByteArray();
+ byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+ int offset = putInt(result, 0, val.scale());
+ putBytes(result, offset, valueBytes, 0, valueBytes.length);
+ return result;
+ }
+
+
+ /**
+ * Converts a byte array to a BigDecimal
+ *
+ * @param bytes
+ * @return the char value
+ */
+ public static BigDecimal toBigDecimal(byte[] bytes) {
+ return toBigDecimal(bytes, 0, bytes.length);
+ }
+
+ /**
+ * Converts a byte array to a BigDecimal value
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return the char value
+ */
+ public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
+ if (bytes == null || length < SIZEOF_INT + 1 ||
+ (offset + length > bytes.length)) {
+ return null;
+ }
+
+ int scale = toInt(bytes, offset);
+ byte[] tcBytes = new byte[length - SIZEOF_INT];
+ System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+ return new BigDecimal(new BigInteger(tcBytes), scale);
+ }
+
+ /**
+ * Put a BigDecimal value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val BigDecimal to write out
+ * @return incremented offset
+ */
+ public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
+ if (bytes == null) {
+ return offset;
+ }
+
+ byte[] valueBytes = val.unscaledValue().toByteArray();
+ byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+ offset = putInt(result, offset, val.scale());
+ return putBytes(result, offset, valueBytes, 0, valueBytes.length);
+ }
+
+ /**
+ * @param vint Integer to make a vint of.
+ * @return Vint as bytes array.
+ */
+ public static byte[] vintToBytes(final long vint) {
+ long i = vint;
+ int size = WritableUtils.getVIntSize(i);
+ byte[] result = new byte[size];
+ int offset = 0;
+ if (i >= -112 && i <= 127) {
+ result[offset] = (byte) i;
+ return result;
+ }
+
+ int len = -112;
+ if (i < 0) {
+ i ^= -1L; // take one's complement'
+ len = -120;
+ }
+
+ long tmp = i;
+ while (tmp != 0) {
+ tmp = tmp >> 8;
+ len--;
+ }
+
+ result[offset++] = (byte) len;
+
+ len = (len < -120) ? -(len + 120) : -(len + 112);
+
+ for (int idx = len; idx != 0; idx--) {
+ int shiftbits = (idx - 1) * 8;
+ long mask = 0xFFL << shiftbits;
+ result[offset++] = (byte) ((i & mask) >> shiftbits);
+ }
+ return result;
+ }
+
+ /**
+ * @param buffer buffer to convert
+ * @return vint bytes as an integer.
+ */
+ public static long bytesToVint(final byte[] buffer) {
+ int offset = 0;
+ byte firstByte = buffer[offset++];
+ int len = WritableUtils.decodeVIntSize(firstByte);
+ if (len == 1) {
+ return firstByte;
+ }
+ long i = 0;
+ for (int idx = 0; idx < len - 1; idx++) {
+ byte b = buffer[offset++];
+ i = i << 8;
+ i = i | (b & 0xFF);
+ }
+ return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
+ }
+
+ /**
+ * Reads a zero-compressed encoded long from input stream and returns it.
+ *
+ * @param buffer Binary array
+ * @param offset Offset into array at which vint begins.
+ * @return deserialized long from stream.
+ * @throws java.io.IOException e
+ */
+ public static long readVLong(final byte[] buffer, final int offset)
+ throws IOException {
+ byte firstByte = buffer[offset];
+ int len = WritableUtils.decodeVIntSize(firstByte);
+ if (len == 1) {
+ return firstByte;
+ }
+ long i = 0;
+ for (int idx = 0; idx < len - 1; idx++) {
+ byte b = buffer[offset + 1 + idx];
+ i = i << 8;
+ i = i | (b & 0xFF);
+ }
+ return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
+ }
+
+ /**
+ * @param left left operand
+ * @param right right operand
+ * @return 0 if equal, < 0 if left is less than right, etc.
+ */
+ public static int compareTo(final byte[] left, final byte[] right) {
+ return LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(left, 0, left.length, right, 0, right.length);
+ }
+
+ /**
+ * Lexicographically compare two arrays.
+ *
+ * @param buffer1 left operand
+ * @param buffer2 right operand
+ * @param offset1 Where to start comparing in the left buffer
+ * @param offset2 Where to start comparing in the right buffer
+ * @param length1 How much to compare from the left buffer
+ * @param length2 How much to compare from the right buffer
+ * @return 0 if equal, < 0 if left is less than right, etc.
+ */
+ public static int compareTo(byte[] buffer1, int offset1, int length1,
+ byte[] buffer2, int offset2, int length2) {
+ return LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
+ }
+
+ interface Comparer<T> {
+ int compareTo(
+ T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
+ );
+ }
+
+ static Comparer<byte[]> lexicographicalComparerJavaImpl() {
+ return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
+ }
+
+ /**
+ * Provides a lexicographical comparer implementation; either a Java
+ * implementation or a faster implementation based on {@link sun.misc.Unsafe}.
+ * <p/>
+ * <p>Uses reflection to gracefully fall back to the Java implementation if
+ * {@code Unsafe} isn't available.
+ */
+ static class LexicographicalComparerHolder {
+ static final String UNSAFE_COMPARER_NAME =
+ LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
+
+ static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
+
+ /**
+ * Returns the Unsafe-using Comparer, or falls back to the pure-Java
+ * implementation if unable to do so.
+ */
+ static Comparer<byte[]> getBestComparer() {
+ try {
+ Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
+
+ // yes, UnsafeComparer does implement Comparer<byte[]>
+ @SuppressWarnings("unchecked")
+ Comparer<byte[]> comparer =
+ (Comparer<byte[]>) theClass.getEnumConstants()[0];
+ return comparer;
+ } catch (Throwable t) { // ensure we really catch *everything*
+ return lexicographicalComparerJavaImpl();
+ }
+ }
+
+ enum PureJavaComparer implements Comparer<byte[]> {
+ INSTANCE;
+
+ @Override
+ public int compareTo(byte[] buffer1, int offset1, int length1,
+ byte[] buffer2, int offset2, int length2) {
+ // Short circuit equal case
+ if (buffer1 == buffer2 &&
+ offset1 == offset2 &&
+ length1 == length2) {
+ return 0;
+ }
+ // Bring WritableComparator code local
+ int end1 = offset1 + length1;
+ int end2 = offset2 + length2;
+ for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
+ int a = (buffer1[i] & 0xff);
+ int b = (buffer2[j] & 0xff);
+ if (a != b) {
+ return a - b;
+ }
+ }
+ return length1 - length2;
+ }
+ }
+
+ enum UnsafeComparer implements Comparer<byte[]> {
+ INSTANCE;
+
+ static final Unsafe theUnsafe;
+
+ /**
+ * The offset to the first element in a byte array.
+ */
+ static final int BYTE_ARRAY_BASE_OFFSET;
+
+ static {
+ theUnsafe = (Unsafe) AccessController.doPrivileged(
+ new PrivilegedAction<Object>() {
+ @Override
+ public Object run() {
+ try {
+ Field f = Unsafe.class.getDeclaredField("theUnsafe");
+ f.setAccessible(true);
+ return f.get(null);
+ } catch (NoSuchFieldException e) {
+ // It doesn't matter what we throw;
+ // it's swallowed in getBestComparer().
+ throw new Error();
+ } catch (IllegalAccessException e) {
+ throw new Error();
+ }
+ }
+ });
+
+ BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
+
+ // sanity check - this should never fail
+ if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
+ throw new AssertionError();
+ }
+ }
+
+ static final boolean littleEndian =
+ ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * Returns true if x1 is less than x2, when both values are treated as
+ * unsigned long.
+ */
+ static boolean lessThanUnsignedLong(long x1, long x2) {
+ return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
+ }
+
+ /**
+ * Returns true if x1 is less than x2, when both values are treated as
+ * unsigned int.
+ */
+ static boolean lessThanUnsignedInt(int x1, int x2) {
+ return (x1 & 0xffffffffL) < (x2 & 0xffffffffL);
+ }
+
+ /**
+ * Returns true if x1 is less than x2, when both values are treated as
+ * unsigned short.
+ */
+ static boolean lessThanUnsignedShort(short x1, short x2) {
+ return (x1 & 0xffff) < (x2 & 0xffff);
+ }
+
+ /**
+ * Checks if Unsafe is available
+ *
+ * @return true, if available, false - otherwise
+ */
+ public static boolean isAvailable() {
+ return theUnsafe != null;
+ }
+
+ /**
+ * Lexicographically compare two arrays.
+ *
+ * @param buffer1 left operand
+ * @param buffer2 right operand
+ * @param offset1 Where to start comparing in the left buffer
+ * @param offset2 Where to start comparing in the right buffer
+ * @param length1 How much to compare from the left buffer
+ * @param length2 How much to compare from the right buffer
+ * @return 0 if equal, < 0 if left is less than right, etc.
+ */
+ @Override
+ public int compareTo(byte[] buffer1, int offset1, int length1,
+ byte[] buffer2, int offset2, int length2) {
+
+ // Short circuit equal case
+ if (buffer1 == buffer2 &&
+ offset1 == offset2 &&
+ length1 == length2) {
+ return 0;
+ }
+ final int minLength = Math.min(length1, length2);
+ final int minWords = minLength / SIZEOF_LONG;
+ final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
+ final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+
+ /*
+ * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
+ * time is no slower than comparing 4 bytes at a time even on 32-bit.
+ * On the other hand, it is substantially faster on 64-bit.
+ */
+ for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
+ long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
+ long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
+ long diff = lw ^ rw;
+ if (littleEndian) {
+ lw = Long.reverseBytes(lw);
+ rw = Long.reverseBytes(rw);
+ }
+ if (diff != 0) {
+ return lessThanUnsignedLong(lw, rw) ? -1 : 1;
+ }
+ }
+ int offset = minWords * SIZEOF_LONG;
+
+ if (minLength - offset >= SIZEOF_INT) {
+ int il = theUnsafe.getInt(buffer1, offset1Adj + offset);
+ int ir = theUnsafe.getInt(buffer2, offset2Adj + offset);
+ if (littleEndian) {
+ il = Integer.reverseBytes(il);
+ ir = Integer.reverseBytes(ir);
+ }
+ if (il != ir) {
+ return lessThanUnsignedInt(il, ir) ? -1 : 1;
+ }
+ offset += SIZEOF_INT;
+ }
+ if (minLength - offset >= SIZEOF_SHORT) {
+ short sl = theUnsafe.getShort(buffer1, offset1Adj + offset);
+ short sr = theUnsafe.getShort(buffer2, offset2Adj + offset);
+ if (littleEndian) {
+ sl = Short.reverseBytes(sl);
+ sr = Short.reverseBytes(sr);
+ }
+ if (sl != sr) {
+ return lessThanUnsignedShort(sl, sr) ? -1 : 1;
+ }
+ offset += SIZEOF_SHORT;
+ }
+ if (minLength - offset == 1) {
+ int a = (buffer1[(int) (offset1 + offset)] & 0xff);
+ int b = (buffer2[(int) (offset2 + offset)] & 0xff);
+ if (a != b) {
+ return a - b;
+ }
+ }
+ return length1 - length2;
+ }
+ }
+ }
+
+ /**
+ * @param left left operand
+ * @param right right operand
+ * @return True if equal
+ */
+ public static boolean equals(final byte[] left, final byte[] right) {
+ // Could use Arrays.equals?
+ //noinspection SimplifiableConditionalExpression
+ if (left == right) return true;
+ if (left == null || right == null) return false;
+ if (left.length != right.length) return false;
+ if (left.length == 0) return true;
+
+ // Since we're often comparing adjacent sorted data,
+ // it's usual to have equal arrays except for the very last byte
+ // so check that first
+ if (left[left.length - 1] != right[right.length - 1]) return false;
+
+ return compareTo(left, right) == 0;
+ }
+
+ public static boolean equals(final byte[] left, int leftOffset, int leftLen,
+ final byte[] right, int rightOffset, int rightLen) {
+ // short circuit case
+ if (left == right &&
+ leftOffset == rightOffset &&
+ leftLen == rightLen) {
+ return true;
+ }
+ // different lengths fast check
+ if (leftLen != rightLen) {
+ return false;
+ }
+ if (leftLen == 0) {
+ return true;
+ }
+
+ // Since we're often comparing adjacent sorted data,
+ // it's usual to have equal arrays except for the very last byte
+ // so check that first
+ if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
+
+ return LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
+ }
+
+
+ /**
+ * @param a left operand
+ * @param buf right operand
+ * @return True if equal
+ */
+ public static boolean equals(byte[] a, ByteBuffer buf) {
+ if (a == null) return buf == null;
+ if (buf == null) return false;
+ if (a.length != buf.remaining()) return false;
+
+ // Thou shalt not modify the original byte buffer in what should be read only operations.
+ ByteBuffer b = buf.duplicate();
+ for (byte anA : a) {
+ if (anA != b.get()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ /**
+ * Return true if the byte array on the right is a prefix of the byte
+ * array on the left.
+ */
+ public static boolean startsWith(byte[] bytes, byte[] prefix) {
+ return bytes != null && prefix != null &&
+ bytes.length >= prefix.length &&
+ LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
+ }
+
+ /**
+ * @param b bytes to hash
+ * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
+ * passed in array. This method is what {@link org.apache.hadoop.io.Text} and
+ * {@link org.apache.hadoop.hbase.io.ImmutableBytesWritable} use calculating hash code.
+ */
+ public static int hashCode(final byte[] b) {
+ return hashCode(b, b.length);
+ }
+
+ /**
+ * @param b value
+ * @param length length of the value
+ * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
+ * passed in array. This method is what {@link org.apache.hadoop.io.Text} and
+ * {@link org.apache.hadoop.hbase.io.ImmutableBytesWritable} use calculating hash code.
+ */
+ public static int hashCode(final byte[] b, final int length) {
+ return WritableComparator.hashBytes(b, length);
+ }
+
+ /**
+ * @param b bytes to hash
+ * @return A hash of <code>b</code> as an Integer that can be used as key in
+ * Maps.
+ */
+ public static Integer mapKey(final byte[] b) {
+ return hashCode(b);
+ }
+
+ /**
+ * @param b bytes to hash
+ * @param length length to hash
+ * @return A hash of <code>b</code> as an Integer that can be used as key in
+ * Maps.
+ */
+ public static Integer mapKey(final byte[] b, final int length) {
+ return hashCode(b, length);
+ }
+
+ /**
+ * @param a lower half
+ * @param b upper half
+ * @return New array that has a in lower half and b in upper half.
+ */
+ public static byte[] add(final byte[] a, final byte[] b) {
+ return add(a, b, EMPTY_BYTE_ARRAY);
+ }
+
+ /**
+ * @param a first third
+ * @param b second third
+ * @param c third third
+ * @return New array made from a, b and c
+ */
+ public static byte[] add(final byte[] a, final byte[] b, final byte[] c) {
+ byte[] result = new byte[a.length + b.length + c.length];
+ System.arraycopy(a, 0, result, 0, a.length);
+ System.arraycopy(b, 0, result, a.length, b.length);
+ System.arraycopy(c, 0, result, a.length + b.length, c.length);
+ return result;
+ }
+
+ /**
+ * @param a array
+ * @param length amount of bytes to grab
+ * @return First <code>length</code> bytes from <code>a</code>
+ */
+ public static byte[] head(final byte[] a, final int length) {
+ if (a.length < length) {
+ return null;
+ }
+ byte[] result = new byte[length];
+ System.arraycopy(a, 0, result, 0, length);
+ return result;
+ }
+
+ /**
+ * @param a array
+ * @param length amount of bytes to snarf
+ * @return Last <code>length</code> bytes from <code>a</code>
+ */
+ public static byte[] tail(final byte[] a, final int length) {
+ if (a.length < length) {
+ return null;
+ }
+ byte[] result = new byte[length];
+ System.arraycopy(a, a.length - length, result, 0, length);
+ return result;
+ }
+
+ /**
+ * @param a array
+ * @param length new array size
+ * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
+ */
+ public static byte[] padHead(final byte[] a, final int length) {
+ byte[] padding = new byte[length];
+ for (int i = 0; i < length; i++) {
+ padding[i] = 0;
+ }
+ return add(padding, a);
+ }
+
+ /**
+ * @param a array
+ * @param length new array size
+ * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
+ */
+ public static byte[] padTail(final byte[] a, final int length) {
+ byte[] padding = new byte[length];
+ for (int i = 0; i < length; i++) {
+ padding[i] = 0;
+ }
+ return add(a, padding);
+ }
+
+ /**
+ * Split passed range. Expensive operation relatively. Uses BigInteger math.
+ * Useful splitting ranges for MapReduce jobs.
+ *
+ * @param a Beginning of range
+ * @param b End of range
+ * @param num Number of times to split range. Pass 1 if you want to split
+ * the range in two; i.e. one split.
+ * @return Array of dividing values
+ */
+ public static byte[][] split(final byte[] a, final byte[] b, final int num) {
+ return split(a, b, false, num);
+ }
+
+ /**
+ * Split passed range. Expensive operation relatively. Uses BigInteger math.
+ * Useful splitting ranges for MapReduce jobs.
+ *
+ * @param a Beginning of range
+ * @param b End of range
+ * @param inclusive Whether the end of range is prefix-inclusive or is
+ * considered an exclusive boundary. Automatic splits are generally exclusive
+ * and manual splits with an explicit range utilize an inclusive end of range.
+ * @param num Number of times to split range. Pass 1 if you want to split
+ * the range in two; i.e. one split.
+ * @return Array of dividing values
+ */
+ public static byte[][] split(final byte[] a, final byte[] b,
+ boolean inclusive, final int num) {
+ byte[][] ret = new byte[num + 2][];
+ int i = 0;
+ Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
+ if (iter == null)
+ return null;
+ for (byte[] elem : iter) {
+ ret[i++] = elem;
+ }
+ return ret;
+ }
+
+ /**
+ * Iterate over keys within the passed range, splitting at an [a,b) boundary.
+ */
+ public static Iterable<byte[]> iterateOnSplits(final byte[] a,
+ final byte[] b, final int num) {
+ return iterateOnSplits(a, b, false, num);
+ }
+
+ /**
+ * Iterate over keys within the passed range.
+ */
+ public static Iterable<byte[]> iterateOnSplits(
+ final byte[] a, final byte[] b, boolean inclusive, final int num) {
+ byte[] aPadded;
+ byte[] bPadded;
+ if (a.length < b.length) {
+ aPadded = padTail(a, b.length - a.length);
+ bPadded = b;
+ } else if (b.length < a.length) {
+ aPadded = a;
+ bPadded = padTail(b, a.length - b.length);
+ } else {
+ aPadded = a;
+ bPadded = b;
+ }
+ if (compareTo(aPadded, bPadded) >= 0) {
+ throw new IllegalArgumentException("b <= a");
+ }
+ if (num <= 0) {
+ throw new IllegalArgumentException("num cannot be <= 0");
+ }
+ byte[] prependHeader = {1, 0};
+ final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
+ final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
+ BigInteger diffBI = stopBI.subtract(startBI);
+ if (inclusive) {
+ diffBI = diffBI.add(BigInteger.ONE);
+ }
+ final BigInteger splitsBI = BigInteger.valueOf(num + 1);
+ if (diffBI.compareTo(splitsBI) < 0) {
+ return null;
+ }
+ final BigInteger intervalBI;
+ try {
+ intervalBI = diffBI.divide(splitsBI);
+ } catch (Exception e) {
+ LOG.error("Exception caught during division", e);
+ return null;
+ }
+
+ final Iterator<byte[]> iterator = new Iterator<byte[]>() {
+ private int i = -1;
+
+ @Override
+ public boolean hasNext() {
+ return i < num + 1;
+ }
+
+ @Override
+ public byte[] next() {
+ i++;
+ if (i == 0) return a;
+ if (i == num + 1) return b;
+
+ BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
+ byte[] padded = curBI.toByteArray();
+ if (padded[1] == 0)
+ padded = tail(padded, padded.length - 2);
+ else
+ padded = tail(padded, padded.length - 1);
+ return padded;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ return new Iterable<byte[]>() {
+ @Override
+ public Iterator<byte[]> iterator() {
+ return iterator;
+ }
+ };
+ }
+
+ /**
+ * @param bytes array to hash
+ * @param offset offset to start from
+ * @param length length to hash
+ */
+ public static int hashCode(byte[] bytes, int offset, int length) {
+ int hash = 1;
+ for (int i = offset; i < offset + length; i++)
+ hash = (31 * hash) + (int) bytes[i];
+ return hash;
+ }
+
+ /**
+ * @param t operands
+ * @return Array of byte arrays made from passed array of Text
+ */
+ public static byte[][] toByteArrays(final String[] t) {
+ byte[][] result = new byte[t.length][];
+ for (int i = 0; i < t.length; i++) {
+ result[i] = Bytes.toBytes(t[i]);
+ }
+ return result;
+ }
+
+ /**
+ * @param column operand
+ * @return A byte array of a byte array where first and only entry is
+ * <code>column</code>
+ */
+ public static byte[][] toByteArrays(final String column) {
+ return toByteArrays(toBytes(column));
+ }
+
+ /**
+ * @param column operand
+ * @return A byte array of a byte array where first and only entry is
+ * <code>column</code>
+ */
+ public static byte[][] toByteArrays(final byte[] column) {
+ byte[][] result = new byte[1][];
+ result[0] = column;
+ return result;
+ }
+
+ /**
+ * Binary search for keys in indexes.
+ *
+ * @param arr array of byte arrays to search for
+ * @param key the key you want to find
+ * @param offset the offset in the key you want to find
+ * @param length the length of the key
+ * @param comparator a comparator to compare.
+ * @return zero-based index of the key, if the key is present in the array.
+ * Otherwise, a value -(i + 1) such that the key is between arr[i -
+ * 1] and arr[i] non-inclusively, where i is in [0, i], if we define
+ * arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
+ * means that this function can return 2N + 1 different values
+ * ranging from -(N + 1) to N - 1.
+ */
+ public static int binarySearch(byte[][] arr, byte[] key, int offset,
+ int length, RawComparator<?> comparator) {
+ int low = 0;
+ int high = arr.length - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ // we have to compare in this order, because the comparator order
+ // has special logic when the 'left side' is a special key.
+ int cmp = comparator.compare(key, offset, length,
+ arr[mid], 0, arr[mid].length);
+ // key lives above the midpoint
+ if (cmp > 0)
+ low = mid + 1;
+ // key lives below the midpoint
+ else if (cmp < 0)
+ high = mid - 1;
+ // BAM. how often does this really happen?
+ else
+ return mid;
+ }
+ return -(low + 1);
+ }
+
+ /**
+ * Bytewise binary increment/deincrement of long contained in byte array
+ * on given amount.
+ *
+ * @param value - array of bytes containing long (length <= SIZEOF_LONG)
+ * @param amount value will be incremented on (deincremented if negative)
+ * @return array of bytes containing incremented long (length == SIZEOF_LONG)
+ */
+ public static byte[] incrementBytes(byte[] value, long amount) {
+ byte[] val = value;
+ if (val.length < SIZEOF_LONG) {
+ // Hopefully this doesn't happen too often.
+ byte[] newvalue;
+ if (val[0] < 0) {
+ newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
+ } else {
+ newvalue = new byte[SIZEOF_LONG];
+ }
+ System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
+ val.length);
+ val = newvalue;
+ } else if (val.length > SIZEOF_LONG) {
+ throw new IllegalArgumentException("Increment Bytes - value too big: " +
+ val.length);
+ }
+ if (amount == 0) return val;
+ if (val[0] < 0) {
+ return binaryIncrementNeg(val, amount);
+ }
+ return binaryIncrementPos(val, amount);
+ }
+
+ /* increment/deincrement for positive value */
+ private static byte[] binaryIncrementPos(byte[] value, long amount) {
+ long amo = amount;
+ int sign = 1;
+ if (amount < 0) {
+ amo = -amount;
+ sign = -1;
+ }
+ for (int i = 0; i < value.length; i++) {
+ int cur = ((int) amo % 256) * sign;
+ amo = (amo >> 8);
+ int val = value[value.length - i - 1] & 0x0ff;
+ int total = val + cur;
+ if (total > 255) {
+ amo += sign;
+ total %= 256;
+ } else if (total < 0) {
+ amo -= sign;
+ }
+ value[value.length - i - 1] = (byte) total;
+ if (amo == 0) return value;
+ }
+ return value;
+ }
+
+ /* increment/deincrement for negative value */
+ private static byte[] binaryIncrementNeg(byte[] value, long amount) {
+ long amo = amount;
+ int sign = 1;
+ if (amount < 0) {
+ amo = -amount;
+ sign = -1;
+ }
+ for (int i = 0; i < value.length; i++) {
+ int cur = ((int) amo % 256) * sign;
+ amo = (amo >> 8);
+ int val = ((~value[value.length - i - 1]) & 0x0ff) + 1;
+ int total = cur - val;
+ if (total >= 0) {
+ amo += sign;
+ } else if (total < -256) {
+ amo -= sign;
+ total %= 256;
+ }
+ value[value.length - i - 1] = (byte) total;
+ if (amo == 0) return value;
+ }
+ return value;
+ }
+
+ /**
+ * Writes a string as a fixed-size field, padded with zeros.
+ */
+ public static void writeStringFixedSize(final DataOutput out, String s,
+ int size) throws IOException {
+ byte[] b = toBytes(s);
+ if (b.length > size) {
+ throw new IOException("Trying to write " + b.length + " bytes (" +
+ toStringBinary(b) + ") into a field of length " + size);
+ }
+
+ out.writeBytes(s);
+ for (int i = 0; i < size - s.length(); ++i)
+ out.writeByte(0);
+ }
+
+ /**
+ * Reads a fixed-size field and interprets it as a string padded with zeros.
+ */
+ public static String readStringFixedSize(final DataInput in, int size)
+ throws IOException {
+ byte[] b = new byte[size];
+ in.readFully(b);
+ int n = b.length;
+ while (n > 0 && b[n - 1] == 0)
+ --n;
+
+ return toString(b, 0, n);
+ }
+
+ /**
+ * Copy the byte array given in parameter and return an instance
+ * of a new byte array with the same length and the same content.
+ *
+ * @param bytes the byte array to duplicate
+ * @return a copy of the given byte array
+ */
+ public static byte[] copy(byte[] bytes) {
+ if (bytes == null) return null;
+ byte[] result = new byte[bytes.length];
+ System.arraycopy(bytes, 0, result, 0, bytes.length);
+ return result;
+ }
+
+ /**
+ * Copy the byte array given in parameter and return an instance
+ * of a new byte array with the same length and the same content.
+ *
+ * @param bytes the byte array to copy from
+ * @param offset
+ * @param length
+ * @return a copy of the given designated byte array
+ */
+ public static byte[] copy(byte[] bytes, final int offset, final int length) {
+ if (bytes == null) return null;
+ byte[] result = new byte[length];
+ System.arraycopy(bytes, offset, result, 0, length);
+ return result;
+ }
+
+ /**
+ * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from
+ * somewhere. (mcorgan)
+ *
+ * @param a Array to search. Entries must be sorted and unique.
+ * @param fromIndex First index inclusive of "a" to include in the search.
+ * @param toIndex Last index exclusive of "a" to include in the search.
+ * @param key The byte to search for.
+ * @return The index of key if found. If not found, return -(index + 1), where negative indicates
+ * "not found" and the "index + 1" handles the "-0" case.
+ */
+ public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
+ int unsignedKey = key & 0xff;
+ int low = fromIndex;
+ int high = toIndex - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int midVal = a[mid] & 0xff;
+
+ if (midVal < unsignedKey) {
+ low = mid + 1;
+ } else if (midVal > unsignedKey) {
+ high = mid - 1;
+ } else {
+ return mid; // key found
+ }
+ }
+ return -(low + 1); // key not found.
+ }
+
+ /**
+ * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding
+ * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes.
+ *
+ * @param input The byte[] to increment.
+ * @return The incremented copy of "in". May be same length or 1 byte longer.
+ */
+ public static byte[] unsignedCopyAndIncrement(final byte[] input) {
+ byte[] copy = copy(input);
+ if (copy == null) {
+ throw new IllegalArgumentException("cannot increment null array");
+ }
+ for (int i = copy.length - 1; i >= 0; --i) {
+ if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum
+ copy[i] = 0;
+ } else {
+ ++copy[i];
+ return copy;
+ }
+ }
+ // we maxed out the array
+ byte[] out = new byte[copy.length + 1];
+ out[0] = 1;
+ System.arraycopy(copy, 0, out, 1, copy.length);
+ return out;
+ }
+
+ public static boolean equals(List<byte[]> a, List<byte[]> b) {
+ if (a == null) {
+ if (b == null) {
+ return true;
+ }
+ return false;
+ }
+ if (b == null) {
+ return false;
+ }
+ if (a.size() != b.size()) {
+ return false;
+ }
+ for (int i = 0; i < a.size(); ++i) {
+ if (!Bytes.equals(a.get(i), b.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ /**
+ * Returns the index of the first appearance of the value {@code target} in
+ * {@code array}.
+ *
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target a primitive {@code byte} value
+ * @return the least index {@code i} for which {@code array[i] == target}, or
+ * {@code -1} if no such index exists.
+ */
+ public static int indexOf(byte[] array, byte target) {
+ for (int i = 0; i < array.length; i++) {
+ if (array[i] == target) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Returns the start position of the first occurrence of the specified {@code
+ * target} within {@code array}, or {@code -1} if there is no such occurrence.
+ * <p/>
+ * <p>More formally, returns the lowest index {@code i} such that {@code
+ * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly
+ * the same elements as {@code target}.
+ *
+ * @param array the array to search for the sequence {@code target}
+ * @param target the array to search for as a sub-sequence of {@code array}
+ */
+ public static int indexOf(byte[] array, byte[] target) {
+ checkNotNull(array, "array");
+ checkNotNull(target, "target");
+ if (target.length == 0) {
+ return 0;
+ }
+
+ outer:
+ for (int i = 0; i < array.length - target.length + 1; i++) {
+ for (int j = 0; j < target.length; j++) {
+ if (array[i + j] != target[j]) {
+ continue outer;
+ }
+ }
+ return i;
+ }
+ return -1;
+ }
+
+ /**
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target a primitive {@code byte} value
+ * @return {@code true} if {@code target} is present as an element anywhere in {@code array}.
+ */
+ public static boolean contains(byte[] array, byte target) {
+ return indexOf(array, target) > -1;
+ }
+
+ /**
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target an array of {@code byte}
+ * @return {@code true} if {@code target} is present anywhere in {@code array}
+ */
+ public static boolean contains(byte[] array, byte[] target) {
+ return indexOf(array, target) > -1;
+ }
+
+ /**
+ * Fill given array with zeros.
+ *
+ * @param b array which needs to be filled with zeros
+ */
+ public static void zero(byte[] b) {
+ zero(b, 0, b.length);
+ }
+
+ /**
+ * Fill given array with zeros at the specified position.
+ *
+ * @param b
+ * @param offset
+ * @param length
+ */
+ public static void zero(byte[] b, int offset, int length) {
+ checkPositionIndex(offset, b.length, "offset");
+ checkArgument(length > 0, "length must be greater than 0");
+ checkPositionIndex(offset + length, b.length, "offset + length");
+ Arrays.fill(b, offset, offset + length, (byte) 0);
+ }
+
+ private static final SecureRandom RNG = new SecureRandom();
+
+ /**
+ * Fill given array with random bytes.
+ *
+ * @param b array which needs to be filled with random bytes
+ */
+ public static void random(byte[] b) {
+ RNG.nextBytes(b);
+ }
+
+ /**
+ * Fill given array with random bytes at the specified position.
+ *
+ * @param b
+ * @param offset
+ * @param length
+ */
+ public static void random(byte[] b, int offset, int length) {
+ checkPositionIndex(offset, b.length, "offset");
+ checkArgument(length > 0, "length must be greater than 0");
+ checkPositionIndex(offset + length, b.length, "offset + length");
+ byte[] buf = new byte[length];
+ RNG.nextBytes(buf);
+ System.arraycopy(buf, 0, b, offset, length);
+ }
+
+ /**
+ * Create a max byte array with the specified max byte count
+ *
+ * @param maxByteCount the length of returned byte array
+ * @return the created max byte array
+ */
+ public static byte[] createMaxByteArray(int maxByteCount) {
+ byte[] maxByteArray = new byte[maxByteCount];
+ for (int i = 0; i < maxByteArray.length; i++) {
+ maxByteArray[i] = (byte) 0xff;
+ }
+ return maxByteArray;
+ }
+
+ /**
+ * Create a byte array which is multiple given bytes
+ *
+ * @param srcBytes
+ * @param multiNum
+ * @return byte array
+ */
+ public static byte[] multiple(byte[] srcBytes, int multiNum) {
+ if (multiNum <= 0) {
+ return new byte[0];
+ }
+ byte[] result = new byte[srcBytes.length * multiNum];
+ for (int i = 0; i < multiNum; i++) {
+ System.arraycopy(srcBytes, 0, result, i * srcBytes.length,
+ srcBytes.length);
+ }
+ return result;
+ }
+
+ /**
+ * Convert a byte array into a hex string
+ *
+ * @param b
+ */
+ public static String toHex(byte[] b) {
+ checkArgument(b.length > 0, "length must be greater than 0");
+ return String.format("%x", new BigInteger(1, b));
+ }
+
+ /**
+ * Create a byte array from a string of hash digits. The length of the
+ * string must be a multiple of 2
+ *
+ * @param hex
+ */
+ public static byte[] fromHex(String hex) {
+ checkArgument(hex.length() > 0, "length must be greater than 0");
+ checkArgument(hex.length() % 2 == 0, "length must be a multiple of 2");
+ // Make sure letters are upper case
+ hex = hex.toUpperCase();
+ byte[] b = new byte[hex.length() / 2];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) +
+ toBinaryFromHex((byte) hex.charAt((2 * i + 1))));
+ }
+ return b;
+ }
+
+}
+
[35/50] [abbrv] incubator-kylin git commit: KYLIN-749 fix ci
Posted by sh...@apache.org.
KYLIN-749 fix ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/033f2aee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/033f2aee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/033f2aee
Branch: refs/heads/streaming-localdict
Commit: 033f2aeec991289147106c27a3b57a511bed0024
Parents: 33f8fee
Author: honma <ho...@ebay.com>
Authored: Tue May 12 15:19:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 12 15:20:57 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../kylin/job/hadoop/invertedindex/IITest.java | 33 +++--
.../metadata/filter/CompareTupleFilter.java | 11 +-
.../metadata/filter/DateConditionModifier.java | 62 +++++++++
.../metadata/filter/IgnoreTsCondition.java | 65 ---------
.../metadata/filter/TsConditionEraser.java | 64 +++++++++
.../metadata/realization/StreamSQLDigest.java | 4 +-
.../kylin/query/enumerator/OLAPEnumerator.java | 20 ++-
.../apache/kylin/query/relnode/OLAPContext.java | 25 ++--
.../kylin/query/relnode/OLAPFilterRel.java | 4 +-
.../apache/kylin/query/test/IIQueryTest.java | 7 +-
.../apache/kylin/query/test/KylinQueryTest.java | 2 +-
.../apache/kylin/query/test/KylinTestBase.java | 33 +----
.../kylin/storage/cache/StorageLayerCache.java | 8 ++
.../DateConditionInplaceModifier.java | 63 ---------
.../DateConditionInplaceModifierTest.java | 34 -----
.../filter/DateConditionModifierTest.java | 33 +++++
.../storage/filter/IgnoreTsConditionTest.java | 133 -------------------
.../storage/filter/TsConditionEraserTest.java | 133 +++++++++++++++++++
19 files changed, 370 insertions(+), 366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index efa1459..c259507 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -63,7 +63,7 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
- System.out.println(time(946684800000L));
+ System.out.println(time(1367798400000L));
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1000L * Integer.MAX_VALUE));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index c3bd556..ecc3c23 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -1,10 +1,9 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import java.io.IOException;
-import java.util.*;
-
-import javax.annotation.Nullable;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -27,9 +26,13 @@ import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.model.*;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.cube.FactDistinctIIColumnsMapper;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
@@ -46,10 +49,9 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
/**
* Created by Hongbin Ma(Binmahone) on 3/26/15.
@@ -82,7 +84,6 @@ public class IITest extends LocalFileMetadataTestCase {
}
});
-
iiRows = Lists.newArrayList();
final Slice slice = new SliceBuilder(iiDesc, (short) 0, true).buildSlice(streamMessages, StringStreamParser.instance);
IIKeyValueCodec codec = new IIKeyValueCodec(slice.getInfo());
@@ -140,8 +141,16 @@ public class IITest extends LocalFileMetadataTestCase {
f1.setParameter(p1);
f1.setReturnType("decimal(19,4)");
+ TblColRef column = ii.getDescriptor().findColumnRef("default.test_kylin_fact", "cal_dt");
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ constantFilter = new ConstantTupleFilter(("2012-08-16"));
+ compareFilter.addChild(constantFilter);
+
EndpointAggregators aggregators = EndpointAggregators.fromFunctions(info, Collections.singletonList(f1));
- CoprocessorFilter filter = CoprocessorFilter.fromFilter(new ClearTextDictionary(info), ConstantTupleFilter.TRUE, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
+ CoprocessorFilter filter = CoprocessorFilter.fromFilter(new ClearTextDictionary(info), compareFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
final Iterator<IIRow> iiRowIterator = iiRows.iterator();
@@ -205,8 +214,8 @@ public class IITest extends LocalFileMetadataTestCase {
}
}, type, projector, aggregators, filter);
+ Assert.assertEquals(2, response.getRowsList().size());
System.out.println(response.getRowsList().size());
-
Set<String> answers = Sets.newHashSet("120.4747", "26.8551");
for (org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) {
byte[] measuresBytes = responseRow.getMeasures().toByteArray();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index fc7fd2c..3fa6962 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -69,6 +69,8 @@ public class CompareTupleFilter extends TupleFilter {
// if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
if (!this.conditionValues.isEmpty() && needSwapOperator()) {
this.operator = SWAP_OP_MAP.get(this.operator);
+ TupleFilter last = this.children.remove(this.children.size() - 1);
+ this.children.add(0, last);
}
} else if (child instanceof ConstantTupleFilter) {
this.conditionValues.addAll(child.getValues());
@@ -90,15 +92,6 @@ public class CompareTupleFilter extends TupleFilter {
return conditionValues;
}
- public void updateValues(Collection<?> values, boolean clear) {
- if (clear) {
- this.conditionValues.clear();
- }
-
- this.conditionValues.addAll(values);
- this.firstCondValue = this.conditionValues.iterator().next();
- }
-
public Object getFirstValue() {
return firstCondValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
new file mode 100644
index 0000000..74e2d71
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
@@ -0,0 +1,62 @@
+package org.apache.kylin.metadata.filter;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class DateConditionModifier implements TupleFilterSerializer.Decorator {
+
+ private IdentityHashMap<TupleFilter, Boolean> dateCompareTupleChildren;
+
+ public DateConditionModifier(TupleFilter root) {
+ this.dateCompareTupleChildren = Maps.newIdentityHashMap();
+ }
+
+ /**
+ * replace filter on timestamp column to null, so that two tuple filter trees can
+ * be compared regardless of the filter condition on timestamp column (In top level where conditions concatenated by ANDs)
+ * @param filter
+ * @return
+ */
+ @Override
+ public TupleFilter onSerialize(TupleFilter filter) {
+
+ if (filter instanceof CompareTupleFilter) {
+ CompareTupleFilter cfilter = (CompareTupleFilter) filter;
+ List<? extends TupleFilter> children = cfilter.getChildren();
+
+ if (children == null || children.size() < 1) {
+ throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
+ }
+
+ TblColRef col = cfilter.getColumn();
+ if (col == null || (!"date".equals(col.getDatatype()))) {
+ return cfilter;
+ }
+
+ for (TupleFilter child : filter.getChildren()) {
+ dateCompareTupleChildren.put(child, true);
+ }
+ }
+
+ if (filter instanceof ConstantTupleFilter && dateCompareTupleChildren.containsKey(filter)) {
+ ConstantTupleFilter constantTupleFilter = (ConstantTupleFilter) filter;
+ Set<String> newValues = Sets.newHashSet();
+
+ for (String value : (Collection<String>) constantTupleFilter.getValues()) {
+ newValues.add(DateFormat.formatToDateStr(Long.valueOf(value)));
+ }
+ return new ConstantTupleFilter(newValues);
+ }
+ return filter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
deleted file mode 100644
index 3ed5d07..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.apache.kylin.metadata.filter;
-
-import java.util.IdentityHashMap;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Maps;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
- */
-public class IgnoreTsCondition implements TupleFilterSerializer.Decorator {
-
- private final TblColRef tsColumn;
- private final TupleFilter root;
-
- private IdentityHashMap<TupleFilter, Boolean> isInTopLevelANDs;
-
- public IgnoreTsCondition(TblColRef tsColumn, TupleFilter root) {
- this.tsColumn = tsColumn;
- this.root = root;
- this.isInTopLevelANDs = Maps.newIdentityHashMap();
- }
-
- /**
- * replace filter on timestamp column to null, so that two tuple filter trees can
- * be compared regardless of the filter condition on timestamp column (In top level where conditions concatenated by ANDs)
- * @param filter
- * @return
- */
- @Override
- public TupleFilter onSerialize(TupleFilter filter) {
-
- if (filter == null)
- return null;
-
- //we just need reference equal
- if (root == filter) {
- isInTopLevelANDs.put(filter, true);
- }
-
- if (isInTopLevelANDs.containsKey(filter)) {
- classifyChildrenByMarking(filter);
-
- if (filter instanceof CompareTupleFilter) {
- TblColRef c = ((CompareTupleFilter) filter).getColumn();
- if (c != null && c.equals(tsColumn)) {
- return null;
- }
- }
- }
-
- return filter;
- }
-
- private void classifyChildrenByMarking(TupleFilter filter) {
- if (filter instanceof LogicalTupleFilter) {
- if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
- for (TupleFilter child : filter.getChildren()) {
- isInTopLevelANDs.put(child, true);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
new file mode 100644
index 0000000..5a7ee40
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/TsConditionEraser.java
@@ -0,0 +1,64 @@
+package org.apache.kylin.metadata.filter;
+
+import com.google.common.collect.Maps;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import java.util.IdentityHashMap;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class TsConditionEraser implements TupleFilterSerializer.Decorator {
+
+ private final TblColRef tsColumn;
+ private final TupleFilter root;
+
+ private IdentityHashMap<TupleFilter, Boolean> isInTopLevelANDs;
+
+ public TsConditionEraser(TblColRef tsColumn, TupleFilter root) {
+ this.tsColumn = tsColumn;
+ this.root = root;
+ this.isInTopLevelANDs = Maps.newIdentityHashMap();
+ }
+
+ /**
+ * replace filter on timestamp column to null, so that two tuple filter trees can
+ * be compared regardless of the filter condition on timestamp column (In top level where conditions concatenated by ANDs)
+ * @param filter
+ * @return
+ */
+ @Override
+ public TupleFilter onSerialize(TupleFilter filter) {
+
+ if (filter == null)
+ return null;
+
+ //we just need reference equal
+ if (root == filter) {
+ isInTopLevelANDs.put(filter, true);
+ }
+
+ if (isInTopLevelANDs.containsKey(filter)) {
+ classifyChildrenByMarking(filter);
+
+ if (filter instanceof CompareTupleFilter) {
+ TblColRef c = ((CompareTupleFilter) filter).getColumn();
+ if (c != null && c.equals(tsColumn)) {
+ return null;
+ }
+ }
+ }
+
+ return filter;
+ }
+
+ private void classifyChildrenByMarking(TupleFilter filter) {
+ if (filter instanceof LogicalTupleFilter) {
+ if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ for (TupleFilter child : filter.getChildren()) {
+ isInTopLevelANDs.put(child, true);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
index fd3bba9..3a1d41a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
@@ -2,7 +2,7 @@ package org.apache.kylin.metadata.realization;
import java.util.Arrays;
-import org.apache.kylin.metadata.filter.IgnoreTsCondition;
+import org.apache.kylin.metadata.filter.TsConditionEraser;
import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
@@ -25,7 +25,7 @@ public class StreamSQLDigest {
this.sqlDigest = sqlDigest;
//must use new instance of IgnoreTsCondition
- IgnoreTsCondition decorator = new IgnoreTsCondition(tsCol, sqlDigest.filter);
+ TsConditionEraser decorator = new TsConditionEraser(tsCol, sqlDigest.filter);
filterSerialized = TupleFilterSerializer.serialize(sqlDigest.filter, decorator, StringCodeSystem.INSTANCE);
int nonFilterHashCode = calculateNonFilterHashCode();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index b7c2748..a5019ab 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,23 +18,25 @@
package org.apache.kylin.query.enumerator;
-import java.util.Map;
-import java.util.Properties;
-
import net.hydromatic.linq4j.Enumerator;
import net.hydromatic.optiq.DataContext;
import net.hydromatic.optiq.jdbc.OptiqConnection;
-
import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.DateConditionModifier;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.coprocessor.DictCodeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Map;
+import java.util.Properties;
+
/**
* @author xjiang
*/
@@ -105,6 +107,10 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// bind dynamic variables
bindVariable(olapContext.filter);
+ //modify date condition
+ olapContext.filter = modifyDateCondition(olapContext.filter);
+ olapContext.resetSQLDigest();
+
// query storage engine
IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization, true);
ITupleIterator iterator = storageEngine.search(olapContext.storageContext, olapContext.getSQLDigest(), olapContext.returnTupleInfo);
@@ -115,6 +121,12 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
return iterator;
}
+ private TupleFilter modifyDateCondition(TupleFilter filter) {
+ DateConditionModifier filterDecorator = new DateConditionModifier(filter);
+ byte[] bytes = TupleFilterSerializer.serialize(filter, filterDecorator, DictCodeSystem.INSTANCE);
+ return TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
+ }
+
private void bindVariable(TupleFilter filter) {
if (filter == null) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 2d89975..6121c80 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -18,25 +18,19 @@
package org.apache.kylin.query.relnode;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.query.schema.OLAPSchema;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeField;
+import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.tuple.TupleInfo;
-import org.apache.kylin.metadata.filter.TupleFilter;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
+
+import java.util.*;
/**
* @author xjiang
@@ -132,6 +126,11 @@ public class OLAPContext {
return sqlDigest;
}
+ public void resetSQLDigest()
+ {
+ this.sqlDigest = null;
+ }
+
public void setReturnTupleInfo(RelDataType rowType, ColumnRowType columnRowType) {
TupleInfo info = new TupleInfo();
List<RelDataTypeField> fieldList = rowType.getFieldList();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 6f8f0f7..71cb2fa 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -29,7 +29,6 @@ import net.hydromatic.optiq.runtime.SqlFunctions;
import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
@@ -260,10 +259,11 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
TupleFilterVisitor visitor = new TupleFilterVisitor(this.columnRowType, context);
context.filter = this.condition.accept(visitor);
- DateConditionInplaceModifier.modify(context.filter);
+
context.filterColumns = collectColumns(context.filter);
}
+
private Set<TblColRef> collectColumns(TupleFilter filter) {
Set<TblColRef> ret = Sets.newHashSet();
collectColumnsRecursively(filter, ret);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index 7a13607..37fc4f3 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -61,7 +61,7 @@ public class IIQueryTest extends KylinQueryTest {
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][]{{"inner"}, {"left"}});
+ return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
}
public IIQueryTest(String joinType) throws Exception {
@@ -74,6 +74,11 @@ public class IIQueryTest extends KylinQueryTest {
}
@Test
+ public void testSingleRunQuery() throws Exception {
+ super.testSingleRunQuery();
+ }
+
+ @Test
public void testDetailedQuery() throws Exception {
execAndCompQuery("src/test/resources/query/sql_ii", null, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index e95da2c..40372ff 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -95,7 +95,7 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql_orderby/query01.sql";
+ String queryFileName = "src/test/resources/query/sql_derived/query07.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
index c1da586..89fb01d 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
@@ -18,31 +18,9 @@
package org.apache.kylin.query.test;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.LogManager;
-
+import com.google.common.io.Files;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.dbunit.Assertion;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -56,8 +34,11 @@ import org.dbunit.ext.h2.H2Connection;
import org.dbunit.ext.h2.H2DataTypeFactory;
import org.junit.Assert;
-import com.google.common.io.Files;
-import org.apache.kylin.common.KylinConfig;
+import java.io.*;
+import java.nio.charset.Charset;
+import java.sql.*;
+import java.util.*;
+import java.util.logging.LogManager;
/**
* Created by hongbin on 2/18/14.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java b/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
new file mode 100644
index 0000000..18a0e02
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StorageLayerCache.java
@@ -0,0 +1,8 @@
+package org.apache.kylin.storage.cache;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/11/15.
+ */
+public interface StorageLayerCache {
+ boolean isCacheEnabled();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
deleted file mode 100644
index 8569058..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 5/7/15.
- *
- * For historical reasons, date types are encoded with DateStrDcitionary
- * rather than TimeStrDictionary, so the constant in CompareTupleFilter should
- * be corrected to "yyyy-MM-dd" format for date types
- */
-public class DateConditionInplaceModifier {
-
- public static void modify(TupleFilter filter) {
-
- if (filter instanceof CompareTupleFilter) {
- CompareTupleFilter cfilter = (CompareTupleFilter) filter;
- List<? extends TupleFilter> children = cfilter.getChildren();
-
- if (children == null || children.size() < 1) {
- throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
- }
-
- if (cfilter.getColumn() == null || (!"date".equals(cfilter.getColumn().getColumnDesc().getTypeName()))) {
- return;
- }
-
- int nonConstantChild = 0;
- boolean firstUpdate = true;
- for (int i = 0; i < children.size(); ++i) {
- if (children.get(i) instanceof ConstantTupleFilter) {
- ConstantTupleFilter constantAsFilter = (ConstantTupleFilter) children.get(i);
- Collection<?> values = constantAsFilter.getValues();
- Collection<Object> newValues = Lists.newArrayList();
- for (Object x : values) {
- newValues.add(x == null ? null : DateFormat.formatToDateStr(Long.parseLong((String) x)));
- }
- cfilter.updateValues(newValues, firstUpdate);
- firstUpdate=false;
- } else {
- nonConstantChild++;
- }
- }
- if (nonConstantChild != 1) {
- throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
- }
-
- return;
-
- } else {
- for (TupleFilter child : filter.getChildren()) {
- modify(child);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
deleted file mode 100644
index 204e8c6..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.apache.kylin.storage.filter;
-
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 5/7/15.
- */
-public class DateConditionInplaceModifierTest extends FilterBaseTest {
- @Test
- public void basicTest() {
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
- TblColRef column = new TblColRef(c1);
-
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
- ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter);
- ConstantTupleFilter constantFilter = null;
- constantFilter = new ConstantTupleFilter("946684800000");
- compareFilter.addChild(constantFilter);
-
- DateConditionInplaceModifier.modify(compareFilter);
- Assert.assertEquals("2000-01-01",compareFilter.getFirstValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
new file mode 100644
index 0000000..29cf7b1
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionModifierTest.java
@@ -0,0 +1,33 @@
+package org.apache.kylin.storage.filter;
+
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.DictCodeSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/7/15.
+ */
+public class DateConditionModifierTest extends FilterBaseTest {
+ @Test
+ public void basicTest() {
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef column = new TblColRef(c1);
+
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ constantFilter = new ConstantTupleFilter("946684800000");
+ compareFilter.addChild(constantFilter);
+
+ DateConditionModifier filterDecorator = new DateConditionModifier(compareFilter);
+ byte[] bytes = TupleFilterSerializer.serialize(compareFilter, filterDecorator, DictCodeSystem.INSTANCE);
+ CompareTupleFilter compareTupleFilter = (CompareTupleFilter) TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
+ Assert.assertEquals("2000-01-01", compareTupleFilter.getFirstValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
deleted file mode 100644
index 52fc5cd..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
+++ /dev/null
@@ -1,133 +0,0 @@
-package org.apache.kylin.storage.filter;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.filter.*;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
- */
-public class IgnoreTsConditionTest extends LocalFileMetadataTestCase {
- IIInstance ii;
- TableRecordInfo tableRecordInfo;
- CoprocessorFilter filter;
- TableDesc factTableDesc;
-
- TblColRef caldt;
- TblColRef siteId;
-
- @Before
- public void setup() throws IOException {
- this.createTestMetadata();
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
- this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
- this.factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
- this.caldt = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
- this.siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");
- }
-
- @After
- public void cleanUp() {
- cleanupTestMetadata();
- }
-
- private TupleFilter mockFilter1(int year) {
- CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
- aFilter.addChild(new ColumnTupleFilter(caldt));
- aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
-
- CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- bFilter.addChild(new ColumnTupleFilter(caldt));
- bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
-
- CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- cFilter.addChild(new ColumnTupleFilter(caldt));
- cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
-
- CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
- dFilter.addChild(new ColumnTupleFilter(siteId));
- dFilter.addChild(new ConstantTupleFilter("0"));
-
- LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
- subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
-
- CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- outFilter.addChild(new ColumnTupleFilter(caldt));
- outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
-
- LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
- root.addChildren(Lists.newArrayList(subRoot, outFilter));
- return root;
- }
-
- private TupleFilter mockFilter2(int year) {
- CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
- aFilter.addChild(new ColumnTupleFilter(caldt));
- aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
-
- CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- bFilter.addChild(new ColumnTupleFilter(caldt));
- bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
-
- CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- cFilter.addChild(new ColumnTupleFilter(caldt));
- cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
-
- CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
- dFilter.addChild(new ColumnTupleFilter(siteId));
- dFilter.addChild(new ConstantTupleFilter("0"));
-
- LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
- subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
-
- CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
- outFilter.addChild(new ColumnTupleFilter(caldt));
- outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
-
- LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
- root.addChildren(Lists.newArrayList(subRoot, outFilter));
- return root;
- }
-
- @Test
- public void positiveTest() {
-
- TupleFilter a = mockFilter1(2000);
- TupleFilter b = mockFilter1(2001);
-
- IgnoreTsCondition decoratorA = new IgnoreTsCondition(caldt, a);
- byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
- IgnoreTsCondition decoratorB = new IgnoreTsCondition(caldt, b);
- byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
- Assert.assertArrayEquals(aBytes, bBytes);
-
- }
-
- @Test
- public void negativeTest()
- {
- TupleFilter a = mockFilter2(2000);
- TupleFilter b = mockFilter2(2001);
-
- IgnoreTsCondition decoratorA = new IgnoreTsCondition(caldt, a);
- byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
- IgnoreTsCondition decoratorB = new IgnoreTsCondition(caldt, b);
- byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
- Assert.assertFalse(Arrays.equals(aBytes,bBytes));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/033f2aee/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
new file mode 100644
index 0000000..c1fff70
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/TsConditionEraserTest.java
@@ -0,0 +1,133 @@
+package org.apache.kylin.storage.filter;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class TsConditionEraserTest extends LocalFileMetadataTestCase {
+ IIInstance ii;
+ TableRecordInfo tableRecordInfo;
+ CoprocessorFilter filter;
+ TableDesc factTableDesc;
+
+ TblColRef caldt;
+ TblColRef siteId;
+
+ @Before
+ public void setup() throws IOException {
+ this.createTestMetadata();
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+ this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
+ this.factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ this.caldt = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
+ this.siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");
+ }
+
+ @After
+ public void cleanUp() {
+ cleanupTestMetadata();
+ }
+
+ private TupleFilter mockFilter1(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ private TupleFilter mockFilter2(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ @Test
+ public void positiveTest() {
+
+ TupleFilter a = mockFilter1(2000);
+ TupleFilter b = mockFilter1(2001);
+
+ TsConditionEraser decoratorA = new TsConditionEraser(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ TsConditionEraser decoratorB = new TsConditionEraser(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertArrayEquals(aBytes, bBytes);
+
+ }
+
+ @Test
+ public void negativeTest()
+ {
+ TupleFilter a = mockFilter2(2000);
+ TupleFilter b = mockFilter2(2001);
+
+ TsConditionEraser decoratorA = new TsConditionEraser(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ TsConditionEraser decoratorB = new TsConditionEraser(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertFalse(Arrays.equals(aBytes,bBytes));
+ }
+}
[22/50] [abbrv] incubator-kylin git commit: KYLIN-755 - extract
copying libs from prepare.sh so that it can be reused
Posted by sh...@apache.org.
KYLIN-755 - extract copying libs from prepare.sh so that it can be reused
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/50c31c65
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/50c31c65
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/50c31c65
Branch: refs/heads/streaming-localdict
Commit: 50c31c6530b46d1e43c821bf46c5ab3b261bd11e
Parents: e349cda
Author: honma <ho...@ebay.com>
Authored: Thu May 7 15:52:57 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:52:57 2015 +0800
----------------------------------------------------------------------
script/prepare.sh | 13 +++----------
script/prepare_libs.sh | 20 ++++++++++++++++++++
2 files changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/50c31c65/script/prepare.sh
----------------------------------------------------------------------
diff --git a/script/prepare.sh b/script/prepare.sh
index b38903b..3e00b25 100755
--- a/script/prepare.sh
+++ b/script/prepare.sh
@@ -9,19 +9,12 @@ then
version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
fi
echo "version ${version}"
+export version
+
+sh script/prepare_libs.sh || { exit 1; }
-echo "copy lib file"
-rm -rf lib
-mkdir lib
cp server/target/kylin-server-${version}.war tomcat/webapps/kylin.war
-cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
-cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
-cp jdbc/target/kylin-jdbc-${version}.jar lib/kylin-jdbc-${version}.jar
-# Copied file becomes 000 for some env (e.g. my Cygwin)
chmod 644 tomcat/webapps/kylin.war
-chmod 644 lib/kylin-job-${version}.jar
-chmod 644 lib/kylin-coprocessor-${version}.jar
-chmod 644 lib/kylin-jdbc-${version}.jar
echo "add js css to war"
if [ ! -d "webapp/dist" ]
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/50c31c65/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/script/prepare_libs.sh b/script/prepare_libs.sh
new file mode 100755
index 0000000..4f5454e
--- /dev/null
+++ b/script/prepare_libs.sh
@@ -0,0 +1,20 @@
+
+#!/bin/sh
+
+if [ -z "$version" ]
+then
+ echo 'version not set'
+ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+fi
+echo "version ${version}"
+
+echo "copy lib file"
+rm -rf lib
+mkdir lib
+cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
+cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
+cp jdbc/target/kylin-jdbc-${version}.jar lib/kylin-jdbc-${version}.jar
+# Copied file becomes 000 for some env (e.g. my Cygwin)
+chmod 644 lib/kylin-job-${version}.jar
+chmod 644 lib/kylin-coprocessor-${version}.jar
+chmod 644 lib/kylin-jdbc-${version}.jar
\ No newline at end of file
[30/50] [abbrv] incubator-kylin git commit: KYLIN-750 Merge cube
segments from HBase table
Posted by sh...@apache.org.
KYLIN-750 Merge cube segments from HBase table
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e008e2cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e008e2cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e008e2cc
Branch: refs/heads/streaming-localdict
Commit: e008e2ccdba67a0b4d0ef065d4d0a368ff09c1e7
Parents: 876ac60
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue May 12 10:26:04 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue May 12 10:26:43 2015 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 2 +
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../apache/kylin/job/cube/CubingJobBuilder.java | 68 +++++--
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 45 ++++-
.../hadoop/cube/FactDistinctColumnsReducer.java | 67 ++++---
.../hadoop/cubev2/MergeCuboidFromHBaseJob.java | 22 ++-
.../cubev2/MergeCuboidFromHBaseMapper.java | 5 +-
.../job/hadoop/cubev2/MergeStatisticsStep.java | 181 +++++++++++++++++++
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 54 ++++--
10 files changed, 368 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 96153fd..a99ca5d 100644
--- a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -53,6 +53,8 @@ abstract public class ResourceStore {
public static final String HYBRID_RESOURCE_ROOT = "/hybrid";
public static final String STREAMING_RESOURCE_ROOT = "/streaming";
public static final String STREAMING_OUTPUT_RESOURCE_ROOT = "/streaming_output";
+ public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
+
private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 50e9c3a..2e5d97a 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -54,6 +54,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert Cuboid Data to HFile";
public static final String STEP_NAME_BULK_LOAD_HFILE = "Load HFile to HBase Table";
public static final String STEP_NAME_MERGE_DICTIONARY = "Merge Cuboid Dictionary";
+ public static final String STEP_NAME_MERGE_STATISTICS = "Merge Cuboid Statistics";
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index a01f462..6593ed2 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -33,6 +33,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.hadoop.cube.*;
import org.apache.kylin.job.hadoop.cubev2.InMemCuboidJob;
import org.apache.kylin.job.hadoop.cubev2.MergeCuboidFromHBaseJob;
+import org.apache.kylin.job.hadoop.cubev2.MergeStatisticsStep;
import org.apache.kylin.job.hadoop.dict.CreateDictionaryJob;
import org.apache.kylin.job.hadoop.hbase.BulkLoadJob;
import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
@@ -112,18 +113,21 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
mergingCuboidPaths.add(getPathToMerge(merging));
}
- if(this.inMemoryCubing()) {
+ AbstractExecutable convertCuboidToHfileStep;
+
+ if (this.inMemoryCubing()) {
// merge from HTable
- addMergeFromHBaseSteps(mergeSegment, mergingSegmentIds, mergingHTables, mergedRootPath, result);
+ convertCuboidToHfileStep = addMergeFromHBaseSteps(mergeSegment, mergingSegmentIds, mergingHTables, mergedRootPath, result);
+
+ // bulk load step
+ result.addTask(createBulkLoadStep(mergeSegment, result.getId()));
} else {
// merge cuboid
addMergeSteps(mergeSegment, mergingSegmentIds, mergingCuboidPaths, mergedRootPath, result);
+ // convert htable
+ convertCuboidToHfileStep = addHTableSteps(mergeSegment, mergedRootPath, result);
}
-
- // convert htable
- AbstractExecutable convertCuboidToHfileStep = addHTableSteps(mergeSegment, mergedRootPath, result);
-
// update cube info
result.addTask(createUpdateCubeInfoAfterMergeStep(mergeSegment, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
@@ -148,15 +152,20 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
mergingHTables.add(merging.getStorageLocationIdentifier());
}
- if(this.inMemoryCubing()) {
+
+ AbstractExecutable convertCuboidToHfileStep;
+ if (this.inMemoryCubing()) {
// merge from HTable
- addMergeFromHBaseSteps(seg, mergingSegmentIds, mergingHTables, mergedCuboidPath, result);
+ convertCuboidToHfileStep = addMergeFromHBaseSteps(seg, mergingSegmentIds, mergingHTables, mergedCuboidPath, result);
+
+ // bulk load step
+ result.addTask(createBulkLoadStep(seg, result.getId()));
} else {
// merge cuboid
addMergeSteps(seg, mergingSegmentIds, mergingCuboidPaths, mergedCuboidPath, result);
+ // convert htable
+ convertCuboidToHfileStep = addHTableSteps(seg, mergedCuboidPath, result);
}
- // convert htable
- AbstractExecutable convertCuboidToHfileStep = addHTableSteps(seg, mergedCuboidPath, result);
// update cube info
result.addTask(createUpdateCubeInfoAfterMergeStep(seg, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
@@ -173,12 +182,22 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
}
- void addMergeFromHBaseSteps(CubeSegment seg, List<String> mergingSegmentIds, List<String> mergingHTables, String mergedCuboidPath, CubingJob result) {
+ AbstractExecutable addMergeFromHBaseSteps(CubeSegment seg, List<String> mergingSegmentIds, List<String> mergingHTables, String mergedCuboidPath, CubingJob result) {
result.addTask(createMergeDictionaryStep(seg, mergingSegmentIds));
+ String mergedStatisticsFolder = getStatisticsPath(seg, result.getId());
+ result.addTask(createMergeStatisticsStep(seg, mergingSegmentIds, mergedStatisticsFolder));
+
+ // create htable step
+ result.addTask(createCreateHTableStep(seg));
+
String formattedTables = StringUtils.join(mergingHTables, ",");
- result.addTask(createMergeCuboidDataFromHBaseStep(seg, formattedTables, mergedCuboidPath));
+ String hFilePath = getHFilePath(seg, result.getId());
+ MapReduceExecutable writeHFileStep = createMergeCuboidDataFromHBaseStep(seg, formattedTables, hFilePath);
+ result.addTask(writeHFileStep);
+
+ return writeHFileStep;
}
Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) {
@@ -209,7 +228,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
}
} else {
// create htable step
- result.addTask(createCreateHTableStep(seg, jobId));
+ result.addTask(createCreateHTableStep(seg));
baseCuboidStep = createInMemCubingStep(seg, intermediateHiveTableLocation, intermediateHiveTableName, cuboidOutputTempPath, result.getId());
result.addTask(baseCuboidStep);
// bulk load step
@@ -225,7 +244,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath));
// create htable step
- result.addTask(createCreateHTableStep(seg, jobId));
+ result.addTask(createCreateHTableStep(seg));
// generate hfiles step
final MapReduceExecutable convertCuboidToHfileStep = createConvertCuboidToHfileStep(seg, cuboidPath, jobId);
result.addTask(convertCuboidToHfileStep);
@@ -415,7 +434,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return rowkeyDistributionStep;
}
- private HadoopShellExecutable createCreateHTableStep(CubeSegment seg, String jobId) {
+ private HadoopShellExecutable createCreateHTableStep(CubeSegment seg) {
HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
StringBuilder cmd = new StringBuilder();
@@ -424,7 +443,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg) + "/part-r-00000");
appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
appendExecCmdParameters(cmd, "statisticsenabled", String.valueOf(inMemoryCubing()));
- appendExecCmdParameters(cmd, "statisticsoutput", getStatisticsPath(seg, jobId));
+ appendExecCmdParameters(cmd, "statisticssamplingpercent", String.valueOf(engineConfig.getConfig().getCubingInMemSamplingPercent()));
createHtableStep.setJobParams(cmd.toString());
createHtableStep.setJobClass(CreateHTableJob.class);
@@ -487,6 +506,18 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return result;
}
+
+ private MergeStatisticsStep createMergeStatisticsStep(CubeSegment seg, List<String> mergingSegmentIds, String mergedStatisticsFolder) {
+ MergeStatisticsStep result = new MergeStatisticsStep();
+ result.setName(ExecutableConstants.STEP_NAME_MERGE_STATISTICS);
+ result.setCubeName(seg.getCubeInstance().getName());
+ result.setSegmentId(seg.getUuid());
+ result.setMergingSegmentIds(mergingSegmentIds);
+ result.setMergedStatisticsPath(mergedStatisticsFolder);
+ return result;
+ }
+
+
private MapReduceExecutable createMergeCuboidDataStep(CubeSegment seg, String inputPath, String outputPath) {
MapReduceExecutable mergeCuboidDataStep = new MapReduceExecutable();
mergeCuboidDataStep.setName(ExecutableConstants.STEP_NAME_MERGE_CUBOID);
@@ -505,7 +536,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
}
- private MapReduceExecutable createMergeCuboidDataFromHBaseStep(CubeSegment seg, String inputPath, String outputPath) {
+ private MapReduceExecutable createMergeCuboidDataFromHBaseStep(CubeSegment seg, String inputPath, String hFilePath) {
MapReduceExecutable mergeCuboidDataStep = new MapReduceExecutable();
mergeCuboidDataStep.setName(ExecutableConstants.STEP_NAME_MERGE_CUBOID);
StringBuilder cmd = new StringBuilder();
@@ -514,7 +545,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
appendExecCmdParameters(cmd, "segmentname", seg.getName());
appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", outputPath);
+ appendExecCmdParameters(cmd, "output", hFilePath);
+ appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
mergeCuboidDataStep.setMapReduceParams(cmd.toString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
index 948f25e..b32baf9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
@@ -89,10 +89,10 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWr
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
Iterator<Integer> it = hllcMap.keySet().iterator();
+ ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
while (it.hasNext()) {
int key = it.next();
HyperLogLogPlusCounter hllc = hllcMap.get(key);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
hllc.writeRegisters(buf);
buf.flip();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 9e4b363..bc10032 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,10 +18,12 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.IOException;
-
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
@@ -31,14 +33,19 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
/**
* @author yangli9
*/
@@ -63,7 +70,8 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
String cubeName = getOptionValue(OPTION_CUBE_NAME);
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String intermediateTable = getOptionValue(OPTION_TABLE_NAME);;
+ String intermediateTable = getOptionValue(OPTION_TABLE_NAME);
+ ;
String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
String statistics_enabled = getOptionValue(OPTION_STATISTICS_ENABLED);
String statistics_output = getOptionValue(OPTION_STATISTICS_OUTPUT);
@@ -73,6 +81,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
// add metadata to distributed cache
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
+ CubeSegment newSegment = cubeInstance.getSegment(segmentName, SegmentStatusEnum.NEW);
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
@@ -82,14 +91,21 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
log.info("Starting: " + job.getJobName());
setJobClasspath(job);
-
+
setupMapper(intermediateTable);
setupReducer(output);
// CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
attachKylinPropsAndMetadata(cubeInstance, job.getConfiguration());
- return waitForCompletion(job);
+ int result = waitForCompletion(job);
+
+ if(Boolean.parseBoolean(statistics_enabled)) {
+ putStatisticsToResourceStore(statistics_output, newSegment);
+ }
+
+ return result;
+
} catch (Exception e) {
logger.error("error in FactDistinctColumnsJob", e);
@@ -105,7 +121,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
HCatInputFormat.setInput(job, dbTableNames[0],
dbTableNames[1]);
-
+
job.setInputFormatClass(HCatInputFormat.class);
job.setMapperClass(FactDistinctHiveColumnsMapper.class);
job.setCombinerClass(FactDistinctColumnsCombiner.class);
@@ -127,6 +143,23 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
deletePath(job.getConfiguration(), output);
}
+ private void putStatisticsToResourceStore(String statisticsFolder, CubeSegment cubeSegment) throws IOException {
+ Path statisticsFilePath = new Path(statisticsFolder, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ FileSystem fs = HadoopUtil.getFileSystem("hdfs:///" + statisticsFilePath);
+ if (!fs.exists(statisticsFilePath))
+ throw new IOException("File " + statisticsFilePath + " does not exists;");
+
+ FSDataInputStream is = fs.open(statisticsFilePath);
+ try {
+ // put the statistics to metadata store
+ String statisticsFileName = ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment.getUuid() + ".seq";
+ ResourceStore rs = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ rs.putResource(statisticsFileName, is, System.currentTimeMillis());
+ } finally {
+ IOUtils.closeStream(is);
+ }
+ }
+
public static void main(String[] args) throws Exception {
FactDistinctColumnsJob job = new FactDistinctColumnsJob();
int exitCode = ToolRunner.run(job, args);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index c671da9..5d29e1b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -24,24 +24,23 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.*;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.mr.KylinReducer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.*;
/**
@@ -53,7 +52,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
private boolean collectStatistics = false;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
- private Map<Long, Long> rowCountInCuboids;
+ // private Map<Long, Long> rowCountInCuboids;
protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
@@ -78,7 +77,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
if (collectStatistics) {
baseCuboidRowCountInMappers = Lists.newArrayList();
- rowCountInCuboids = Maps.newHashMap();
+// rowCountInCuboids = Maps.newHashMap();
cuboidHLLMap = Maps.newHashMap();
SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
}
@@ -113,23 +112,23 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
// for hll
long cuboidId = 0 - key.get();
- for (Text value : values) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
- ByteArray byteArray = new ByteArray(value.getBytes());
- hll.readRegisters(byteArray.asBuffer());
+ for (Text value : values) {
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
+ ByteArray byteArray = new ByteArray(value.getBytes());
+ hll.readRegisters(byteArray.asBuffer());
- totalRowsBeforeMerge += hll.getCountEstimate();
+ totalRowsBeforeMerge += hll.getCountEstimate();
- if (cuboidId == baseCuboidId) {
- baseCuboidRowCountInMappers.add(hll.getCountEstimate());
- }
+ if (cuboidId == baseCuboidId) {
+ baseCuboidRowCountInMappers.add(hll.getCountEstimate());
+ }
- if (cuboidHLLMap.get(cuboidId) != null) {
- cuboidHLLMap.get(cuboidId).merge(hll);
- } else {
- cuboidHLLMap.put(cuboidId, hll);
- }
+ if (cuboidHLLMap.get(cuboidId) != null) {
+ cuboidHLLMap.get(cuboidId).merge(hll);
+ } else {
+ cuboidHLLMap.put(cuboidId, hll);
}
+ }
}
}
@@ -139,12 +138,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
//output the hll info;
if (collectStatistics) {
- for (Long cuboidId : cuboidHLLMap.keySet()) {
- rowCountInCuboids.put(cuboidId, cuboidHLLMap.get(cuboidId).getCountEstimate());
- }
-
writeMapperAndCuboidStatistics(context); // for human check
- writeCuboidStatistics(context); // for CreateHTableJob
+ writeCuboidStatistics(context.getConfiguration(), statisticsOutput, cuboidHLLMap); // for CreateHTableJob
}
}
@@ -157,7 +152,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
String msg;
List<Long> allCuboids = new ArrayList<Long>();
- allCuboids.addAll(rowCountInCuboids.keySet());
+ allCuboids.addAll(cuboidHLLMap.keySet());
Collections.sort(allCuboids);
msg = "Total cuboid number: \t" + allCuboids.size();
@@ -175,8 +170,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
long grantTotal = 0;
for (long i : allCuboids) {
- grantTotal += rowCountInCuboids.get(i);
- msg = "Cuboid " + i + " row count is: \t " + rowCountInCuboids.get(i);
+ grantTotal += cuboidHLLMap.get(i).getCountEstimate();
+ msg = "Cuboid " + i + " row count is: \t " + cuboidHLLMap.get(i).getCountEstimate();
writeLine(out, msg);
}
@@ -200,19 +195,23 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
}
- private void writeCuboidStatistics(Context context) throws IOException {
- Configuration conf = context.getConfiguration();
- Path seqFilePath = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ public static void writeCuboidStatistics(Configuration conf, String outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap) throws IOException {
+ Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
SequenceFile.Writer writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class),
- SequenceFile.Writer.valueClass(LongWritable.class));
+ SequenceFile.Writer.valueClass(BytesWritable.class));
List<Long> allCuboids = new ArrayList<Long>();
- allCuboids.addAll(rowCountInCuboids.keySet());
+ allCuboids.addAll(cuboidHLLMap.keySet());
Collections.sort(allCuboids);
+
+ ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
try {
for (long i : allCuboids) {
- writer.append(new LongWritable(i), new LongWritable((long) (rowCountInCuboids.get(i) *100 / SAMPING_PERCENTAGE)));
+ valueBuf.clear();
+ cuboidHLLMap.get(i).writeRegisters(valueBuf);
+ valueBuf.flip();
+ writer.append(new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()));
}
} finally {
writer.close();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
index b49af1a..4f50279 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
@@ -22,7 +22,11 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
@@ -55,10 +59,12 @@ public class MergeCuboidFromHBaseJob extends CuboidJob {
options.addOption(OPTION_SEGMENT_NAME);
options.addOption(OPTION_INPUT_PATH);
options.addOption(OPTION_OUTPUT_PATH);
+ options.addOption(OPTION_HTABLE_NAME);
parseOptions(options, args);
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
String segmentName = getOptionValue(OPTION_SEGMENT_NAME).toUpperCase();
+ String htableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(cubeName);
@@ -87,15 +93,18 @@ public class MergeCuboidFromHBaseJob extends CuboidJob {
scans.add(scan);
}
- TableMapReduceUtil.initTableMapperJob(scans, MergeCuboidFromHBaseMapper.class, Text.class,
+ TableMapReduceUtil.initTableMapperJob(scans, MergeCuboidFromHBaseMapper.class, ImmutableBytesWritable.class,
Text.class, job);
+ /*
// Reducer - only one
job.setReducerClass(CuboidReducer.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
+*/
+
// set job configuration
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
@@ -103,7 +112,16 @@ public class MergeCuboidFromHBaseJob extends CuboidJob {
// add metadata to distributed cache
attachKylinPropsAndMetadata(cube, job.getConfiguration());
- setReduceTaskNum(job, config, cubeName, 0);
+
+ HTable htable = new HTable(conf, htableName);
+ HFileOutputFormat.configureIncrementalLoad(job,
+ htable);
+
+
+ // set Reducer; This need be after configureIncrementalLoad, to overwrite the default reducer class
+ job.setReducerClass(InMemCuboidReducer.class);
+ job.setOutputKeyClass(ImmutableBytesWritable.class);
+ job.setOutputValueClass(KeyValue.class);
this.deletePath(job.getConfiguration(), output);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
index 7c673da..59fa03d 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
@@ -59,7 +59,7 @@ import java.util.List;
/**
* @author shaoshi
*/
-public class MergeCuboidFromHBaseMapper extends TableMapper<Text, Text> {
+public class MergeCuboidFromHBaseMapper extends TableMapper<ImmutableBytesWritable, Text> {
private KylinConfig config;
private String cubeName;
@@ -71,7 +71,8 @@ public class MergeCuboidFromHBaseMapper extends TableMapper<Text, Text> {
private CubeSegment sourceCubeSegment;// Must be unique during a mapper's
// life cycle
- private Text outputKey = new Text();
+// private Text outputKey = new Text();
+ private ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
private byte[] newKeyBuf;
private RowKeySplitter rowKeySplitter;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
new file mode 100644
index 0000000..5fe3f96
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cubev2;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.constant.BatchConstants;
+import org.apache.kylin.job.exception.ExecuteException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.job.hadoop.cube.FactDistinctColumnsReducer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class MergeStatisticsStep extends AbstractExecutable {
+
+ private static final String CUBE_NAME = "cubeName";
+ private static final String SEGMENT_ID = "segmentId";
+ private static final String MERGING_SEGMENT_IS = "mergingSegmentIds";
+ private static final String MERGED_STATISTICS_PATH = "mergedStatisticsPath";
+ protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
+
+ public MergeStatisticsStep() {
+ super();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ KylinConfig kylinConf = context.getConfig();
+ final CubeManager mgr = CubeManager.getInstance(kylinConf);
+ final CubeInstance cube = mgr.getCube(getCubeName());
+ final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
+
+ Configuration conf = new Configuration();
+ ResourceStore rs = ResourceStore.getStore(kylinConf);
+ try {
+
+ for (String segmentId : this.getMergingSegmentIds()) {
+ String fileKey = ResourceStore.CUBE_STATISTICS_ROOT + "/" + getCubeName() + "/" + segmentId + ".seq";
+ InputStream is = rs.getResource(fileKey);
+ File tempFile = null;
+ FileOutputStream tempFileStream = null;
+ try {
+ tempFile = File.createTempFile(segmentId, ".seq");
+ tempFileStream = new FileOutputStream(tempFile);
+ org.apache.commons.io.IOUtils.copy(is, tempFileStream);
+ } finally {
+ IOUtils.closeStream(is);
+ IOUtils.closeStream(tempFileStream);
+ }
+
+ FileSystem fs = HadoopUtil.getFileSystem("file:///" + tempFile.getAbsolutePath());
+ SequenceFile.Reader reader = null;
+ try {
+ reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf);
+ LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ while (reader.next(key, value)) {
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
+ ByteArray byteArray = new ByteArray(value.getBytes());
+ hll.readRegisters(byteArray.asBuffer());
+
+ if (cuboidHLLMap.get(key.get()) != null) {
+ cuboidHLLMap.get(key.get()).merge(hll);
+ } else {
+ cuboidHLLMap.put(key.get(), hll);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ }
+
+ FactDistinctColumnsReducer.writeCuboidStatistics(conf, getMergedStatisticsPath(), cuboidHLLMap);
+ Path statisticsFilePath = new Path(getMergedStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+ FileSystem fs = statisticsFilePath.getFileSystem(conf);
+ FSDataInputStream is = fs.open(statisticsFilePath);
+ try {
+ // put the statistics to metadata store
+ String statisticsFileName = ResourceStore.CUBE_STATISTICS_ROOT + "/" + getCubeName() + "/" + newSegment.getUuid() + ".seq";
+ rs.putResource(statisticsFileName, is, System.currentTimeMillis());
+ } finally {
+ IOUtils.closeStream(is);
+ }
+
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+ } catch (IOException e) {
+ logger.error("fail to merge cuboid statistics", e);
+ return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
+ }
+ }
+
+
+ public void setCubeName(String cubeName) {
+ this.setParam(CUBE_NAME, cubeName);
+ }
+
+ private String getCubeName() {
+ return getParam(CUBE_NAME);
+ }
+
+ public void setSegmentId(String segmentId) {
+ this.setParam(SEGMENT_ID, segmentId);
+ }
+
+ private String getSegmentId() {
+ return getParam(SEGMENT_ID);
+ }
+
+ public void setMergingSegmentIds(List<String> ids) {
+ setParam(MERGING_SEGMENT_IS, StringUtils.join(ids, ","));
+ }
+
+ private List<String> getMergingSegmentIds() {
+ final String ids = getParam(MERGING_SEGMENT_IS);
+ if (ids != null) {
+ final String[] splitted = StringUtils.split(ids, ",");
+ ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
+ for (String id : splitted) {
+ result.add(id);
+ }
+ return result;
+ } else {
+ return Collections.emptyList();
+ }
+ }
+
+ public void setMergedStatisticsPath(String path) {
+ setParam(MERGED_STATISTICS_PATH, path);
+ }
+
+ private String getMergedStatisticsPath() {
+ return getParam(MERGED_STATISTICS_PATH);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e008e2cc/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index cbe8c10..6e884d5 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -33,12 +33,16 @@ import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -46,7 +50,6 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.job.tools.DeployCoprocessorCLI;
import org.apache.kylin.job.tools.LZOSupportnessChecker;
@@ -55,7 +58,10 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -75,6 +81,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
CubeDesc cubeDesc = null;
String segmentName = null;
KylinConfig kylinConfig;
+ private int SAMPING_PERCENTAGE = 5;
@Override
public int run(String[] args) throws Exception {
@@ -85,12 +92,15 @@ public class CreateHTableJob extends AbstractHadoopJob {
options.addOption(OPTION_PARTITION_FILE_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_STATISTICS_ENABLED);
- options.addOption(OPTION_STATISTICS_OUTPUT);
+ options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT);
parseOptions(options, args);
Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
boolean statistics_enabled = Boolean.parseBoolean(getOptionValue(OPTION_STATISTICS_ENABLED));
- Path statisticsFilePath = new Path(getOptionValue(OPTION_STATISTICS_OUTPUT), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
+
+ String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT);
+
+ SAMPING_PERCENTAGE = Integer.parseInt(statistics_sampling_percent);
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -133,7 +143,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
byte[][] splitKeys;
if (statistics_enabled) {
- splitKeys = getSplitsFromCuboidStatistics(conf, statisticsFilePath);
+ splitKeys = getSplitsFromCuboidStatistics(conf);
} else {
splitKeys = getSplits(conf, partitionFilePath);
}
@@ -196,7 +206,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
@SuppressWarnings("deprecation")
- protected byte[][] getSplitsFromCuboidStatistics(Configuration conf, Path statisticsFilePath) throws IOException {
+ protected byte[][] getSplitsFromCuboidStatistics(Configuration conf) throws IOException {
List<Integer> rowkeyColumnSize = Lists.newArrayList();
CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
@@ -217,20 +227,33 @@ public class CreateHTableJob extends AbstractHadoopJob {
Map<Long, Long> cuboidSizeMap = Maps.newHashMap();
long totalSizeInM = 0;
- SequenceFile.Reader reader = null;
-
- FileSystem fs = statisticsFilePath.getFileSystem(conf);
- if (fs.exists(statisticsFilePath) == false) {
- System.err.println("Path " + statisticsFilePath + " not found, no region split, HTable will be one region");
- return null;
+ ResourceStore rs = ResourceStore.getStore(kylinConfig);
+ String fileKey = ResourceStore.CUBE_STATISTICS_ROOT + "/" + cube.getName() + "/" + cubeSegment.getUuid() + ".seq";
+ InputStream is = rs.getResource(fileKey);
+ File tempFile = null;
+ FileOutputStream tempFileStream = null;
+ try {
+ tempFile = File.createTempFile(cubeSegment.getUuid(), ".seq");
+ tempFileStream = new FileOutputStream(tempFile);
+ org.apache.commons.io.IOUtils.copy(is, tempFileStream);
+ } finally {
+ IOUtils.closeStream(is);
+ IOUtils.closeStream(tempFileStream);
}
+ FileSystem fs = HadoopUtil.getFileSystem("file:///" +tempFile.getAbsolutePath());
+ SequenceFile.Reader reader = null;
try {
- reader = new SequenceFile.Reader(fs, statisticsFilePath, conf);
+ reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf);
LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- LongWritable value = (LongWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {
- cuboidSizeMap.put(key.get(), value.get());
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
+ ByteArray byteArray = new ByteArray(value.getBytes());
+ hll.readRegisters(byteArray.asBuffer());
+
+ cuboidSizeMap.put(key.get(), hll.getCountEstimate() * 100 / SAMPING_PERCENTAGE);
+
}
} catch (Exception e) {
e.printStackTrace();
@@ -263,7 +286,6 @@ public class CreateHTableJob extends AbstractHadoopJob {
List<Long> regionSplit = Lists.newArrayList();
-
long size = 0;
int regionIndex = 0;
int cuboidCount = 0;
[18/50] [abbrv] incubator-kylin git commit: KYLIN-749 pending no how
to deal with time
Posted by sh...@apache.org.
KYLIN-749 pending no how to deal with time
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/05e0bada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/05e0bada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/05e0bada
Branch: refs/heads/streaming-localdict
Commit: 05e0badad3a4263a64d37eece14507fd334af682
Parents: dcf0a21
Author: honma <ho...@ebay.com>
Authored: Wed May 6 12:35:44 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:04:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 19 +++++-----
.../apache/kylin/dict/DictionaryGenerator.java | 17 +++++----
.../apache/kylin/dict/NumberDictionaryTest.java | 20 +++++------
.../invertedindex/index/TableRecordInfo.java | 6 ++--
.../invertedindex/model/IIKeyValueCodec.java | 12 ++++---
.../kylin/job/dataGen/FactTableGenerator.java | 23 ++++++------
.../apache/kylin/metadata/model/DataType.java | 5 +++
.../kylin/query/relnode/OLAPFilterRel.java | 38 +++++---------------
.../apache/kylin/query/test/KylinQueryTest.java | 23 ++++++------
9 files changed, 76 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 33f6141..db80934 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -18,19 +18,20 @@
package org.apache.kylin.common.util;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ranges;
import org.apache.commons.configuration.ConfigurationException;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+
/**
* Created by honma on 10/17/14.
* <p/>
@@ -62,6 +63,8 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
+ System.out.println(Long.parseLong("0032"));
+ System.out.println(time(1000L * Integer.MAX_VALUE));
System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(3, 10)));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index c4acf7f..59d7bb3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -36,14 +36,14 @@ import java.util.*;
/**
* @author yangli9
*/
-@SuppressWarnings({"rawtypes", "unchecked"})
+@SuppressWarnings({ "rawtypes", "unchecked" })
public class DictionaryGenerator {
private static final int DICT_MAX_CARDINALITY = 2000000; // 2 million
private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
- private static final String[] DATE_PATTERNS = new String[]{"yyyy-MM-dd"};
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd" };
public static Dictionary<?> buildDictionaryFromValueList(DictionaryInfo info, Collection<byte[]> values) {
Dictionary<?> dict = buildDictionaryFromValueList(DataType.getInstance(info.getDataType()), values);
@@ -60,7 +60,7 @@ public class DictionaryGenerator {
// build dict, case by data type
if (dataType.isDateTimeFamily()) {
- dict = buildDateStrDict(values, baseId, nSamples, samples);
+ dict = buildDateTimeDict(values, baseId, nSamples, samples);
} else if (dataType.isNumberFamily()) {
dict = buildNumberDict(values, baseId, nSamples, samples);
} else {
@@ -77,7 +77,7 @@ public class DictionaryGenerator {
}
logger.debug("Dictionary value samples: " + buf.toString());
logger.debug("Dictionary cardinality " + dict.getSize());
- if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
+ if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- cardinality: " + values.size());
}
return dict;
@@ -116,7 +116,7 @@ public class DictionaryGenerator {
return buildDictionaryFromValueList(info, values);
}
- private static Dictionary buildDateStrDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ private static Dictionary buildDateTimeDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
final int BAD_THRESHOLD = 2;
String matchPattern = null;
@@ -142,10 +142,13 @@ public class DictionaryGenerator {
}
}
}
- if (matchPattern != null)
+ if (matchPattern != null) {
return new DateStrDictionary(matchPattern, baseId);
+ }
}
- throw new IllegalStateException("Unrecognized datetime value");
+
+ //FIXME: except for date type, all other date time family types are treated as string
+ return buildStringDict(values, baseId, nSamples, samples);
}
private static Dictionary buildStringDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index f9af244..405fece 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -18,22 +18,17 @@
package org.apache.kylin.dict;
-import static org.junit.Assert.*;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import java.math.BigDecimal;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
/**
* @author yangli9
@@ -70,6 +65,7 @@ public class NumberDictionaryTest {
checkCodec("-12345.123", "-9999999999987654.876;");
checkCodec("0", "00000000000000000");
checkCodec("0.0", "00000000000000000.0");
+ checkCodec("123456789123456789", "-9999999999987654;");
}
private void checkCodec(String number, String code) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index f2d847c..c57e91f 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -18,8 +18,6 @@
package org.apache.kylin.invertedindex.index;
-import java.util.List;
-
import org.apache.kylin.common.util.Array;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
@@ -28,6 +26,8 @@ import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.List;
+
/**
* @author yangli9
* <p/>
@@ -80,7 +80,7 @@ public class TableRecordInfo {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 10;
+ lengths[i] = 23;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 7c1b802..4300140 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -20,18 +20,22 @@ package org.apache.kylin.invertedindex.model;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.index.ColumnValueContainer;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import java.io.*;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
/**
* @author yangli9
@@ -139,7 +143,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 10;
+ lengths[i] = 23;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 9bcaf1b..7cafea6 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job.dataGen;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Array;
@@ -32,12 +25,15 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.*;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
/**
* Created by hongbin on 5/20/14.
@@ -347,6 +343,9 @@ public class FactTableGenerator {
return String.format("%.4f", r.nextDouble() * (high - low) + low);
} else if (type.isDateTimeFamily()) {
+ if (!type.isDate()) {
+ throw new RuntimeException("Does not support " + type);
+ }
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
Date start = format.parse(range.get(0));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index f60db03..607b547 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -203,6 +203,11 @@ public class DataType {
return DATETIME_FAMILY.contains(name);
}
+ public boolean isDate()
+ {
+ return name.equals("date");
+ }
+
public boolean isTinyInt() {
return name.equals("tinyint");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 16f2562..f15078a 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,50 +18,28 @@
package org.apache.kylin.query.relnode;
-import java.util.*;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import net.hydromatic.optiq.rules.java.EnumerableConvention;
import net.hydromatic.optiq.rules.java.EnumerableRel;
import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
import net.hydromatic.optiq.rules.java.JavaRules.EnumerableCalcRel;
import net.hydromatic.optiq.runtime.SqlFunctions;
-
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.relopt.*;
import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexBuilder;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexDynamicParam;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexLocalRef;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexProgram;
-import org.eigenbase.rex.RexProgramBuilder;
-import org.eigenbase.rex.RexVisitorImpl;
+import org.eigenbase.rex.*;
import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.util.NlsString;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.filter.CaseTupleFilter;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.DynamicTupleFilter;
-import org.apache.kylin.metadata.filter.ExtractTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import java.util.*;
/**
* @author xjiang
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 086d1f1..6a20bdb 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -18,13 +18,6 @@
package org.apache.kylin.query.test;
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.sql.DriverManager;
-import java.util.List;
-import java.util.Properties;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -40,6 +33,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.File;
+import java.sql.DriverManager;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
@Ignore("KylinQueryTest is contained by CombinationTest")
public class KylinQueryTest extends KylinTestBase {
@@ -95,11 +95,13 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql_subquery/query04.sql";
+ String queryFileName = "src/test/resources/query/sample.sql";
File sqlFile = new File(queryFileName);
- runSQL(sqlFile, true, true);
- runSQL(sqlFile, true, false);
+ if (sqlFile.exists()) {
+ runSQL(sqlFile, true, true);
+ runSQL(sqlFile, true, false);
+ }
}
@Test
@@ -208,7 +210,6 @@ public class KylinQueryTest extends KylinTestBase {
}
}
-
@Test
public void testDynamicQuery() throws Exception {
execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);
[36/50] [abbrv] incubator-kylin git commit: KYLIN-728 use more
accurate memory estimation to reduce disk io, accelerate build speed
Posted by sh...@apache.org.
KYLIN-728 use more accurate memory estimation to reduce disk io, accelerate build speed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9e58ae87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9e58ae87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9e58ae87
Branch: refs/heads/streaming-localdict
Commit: 9e58ae8795bfc44b38f04dff47f0d9be4c221e02
Parents: 0d87e8f
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue May 12 16:46:03 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue May 12 16:46:03 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cubev2/InMemCubeBuilder.java | 40 +++++++++-----------
.../kylin/storage/gridtable/GTComboStore.java | 2 -
.../storage/gridtable/diskstore/FileSystem.java | 2 +
.../gridtable/diskstore/GTDiskStore.java | 9 ++++-
.../gridtable/diskstore/HadoopFileSystem.java | 14 ++++++-
.../gridtable/diskstore/LocalFileSystem.java | 12 +++++-
6 files changed, 48 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index 87ad2d3..f869caa 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -69,6 +69,7 @@ import java.util.concurrent.BlockingQueue;
@SuppressWarnings("rawtypes")
public class InMemCubeBuilder implements Runnable {
+ //estimation of (size of aggregation cache) / (size of mem store)
private static final double AGGREGATION_CACHE_FACTOR = 3;
private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class);
@@ -380,48 +381,41 @@ public class InMemCubeBuilder implements Runnable {
record.setValues(recordValues);
}
- private boolean checkMemory(long threshold) {
+ private long checkMemory(long threshold) {
final long freeMemory = Runtime.getRuntime().freeMemory();
- logger.info("available memory:" + (freeMemory>>10) + " KB");
- if (freeMemory >= threshold) {
- logger.info("no need to flush to disk");
- return true;
- } else {
- return false;
- }
+ logger.info("available memory:" + (freeMemory >> 10) + " KB, memory needed:" + (threshold >> 10) + " KB");
+ return freeMemory - threshold;
}
private boolean gc(TreeNode<GridTable> parentNode) {
final long parentCuboidMem = SizeOfUtil.deepSizeOf(parentNode.data.getStore());
long threshold = (long) (parentCuboidMem * (AGGREGATION_CACHE_FACTOR + 1));
- logger.info((threshold >> 10) + " KB is needed to create " + parentNode.id + "'s child");
- if (checkMemory(threshold)) {
- return true;
- }
final List<TreeNode<GridTable>> gridTables = parentNode.getAncestorList();
+ long memoryLeft = checkMemory(threshold);
for (TreeNode<GridTable> gridTable : gridTables) {
- logger.info("wait 10 seconds for gc");
- try {
- Thread.sleep(10 * 1000);
- } catch (InterruptedException e) {
- logger.error("this should not happen", e);
- }
- if (checkMemory(threshold)) {
+ if (memoryLeft >= 0) {
return true;
} else {
logger.info("memory is low, try to select one node to flush to disk from:" + StringUtils.join(",", gridTables));
final IGTStore store = gridTable.data.getStore();
assert store instanceof GTComboStore;
if (store.memoryUsage() > 0) {
- logger.info("cuboid id:" + gridTable.id + " selected, memory used:" + (SizeOfUtil.deepSizeOf(store)>>10) + " KB");
+ final long storeSize = SizeOfUtil.deepSizeOf(store);
+ memoryLeft += storeSize;
+ logger.info("cuboid id:" + gridTable.id + " selected, memory used:" + (storeSize >> 10) + " KB");
long t = System.currentTimeMillis();
((GTComboStore) store).switchToDiskStore();
logger.info("switch to disk store cost:" + (System.currentTimeMillis() - t) + "ms");
}
}
}
- logger.info("no store has been flushed to disk");
- return true;
+ if (memoryLeft >= 0) {
+ return true;
+ } else {
+ logger.warn("all ancestor nodes of " + parentNode.id + " has been flushed to disk, memory is still insufficient, usually due to jvm gc not finished, forced to use memory store");
+ return true;
+ }
+
}
private void createNDCuboidGT(SimpleGridTableTree parentNode, long parentCuboidId, long cuboidId) throws IOException {
@@ -431,7 +425,7 @@ public class InMemCubeBuilder implements Runnable {
if (parentNode.data.getStore().memoryUsage() <= 0) {
long t = System.currentTimeMillis();
((GTComboStore) parentNode.data.getStore()).switchToMemStore();
- logger.info("switch to mem store cost:" + (System.currentTimeMillis() - t) + "ms");
+ logger.info("node " + parentNode.id + " switch to mem store cost:" + (System.currentTimeMillis() - t) + "ms");
}
boolean inMem = gc(parentNode);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
index c7d0c2b..a6476ca 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -62,7 +62,6 @@ public class GTComboStore implements IGTStore {
logger.error("fail to switch to mem store", e);
throw new RuntimeException(e);
}
- logger.info("switch to mem store");
}
public void switchToDiskStore() {
@@ -77,7 +76,6 @@ public class GTComboStore implements IGTStore {
logger.error("fail to switch to disk store", e);
throw new RuntimeException(e);
}
- logger.info("switch to disk store");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
index 2ab2c7e..bebc1a2 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
@@ -12,6 +12,8 @@ interface FileSystem {
boolean delete(String path);
+ void deleteOnExit(String path);
+
boolean createDirectory(String path);
boolean createFile(String path);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
index f48fce3..fec0d13 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -33,6 +33,7 @@ public class GTDiskStore implements IGTStore {
this.identifier = generateIdentifier(fileSystem);
logger.info("disk store created, identifier:" + identifier);
this.writer = new DiskStoreWriter(fileSystem.getWriter(getRowBlockFile(identifier)));
+ deleteTmpFilesOnExit();
}
private String generateIdentifier(FileSystem fs) {
@@ -153,8 +154,12 @@ public class GTDiskStore implements IGTStore {
} catch (Exception e) {
logger.error("error to close writer", e);
}
- fileSystem.delete(getRowBlockFile(identifier));
- fileSystem.delete(getRootDirectory(identifier));
+ deleteTmpFilesOnExit();
+ }
+
+ private void deleteTmpFilesOnExit() {
+ fileSystem.deleteOnExit(getRowBlockFile(identifier));
+ fileSystem.deleteOnExit(getRootDirectory(identifier));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
index e1efd1b..7a6450d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
@@ -12,13 +12,13 @@ import java.io.OutputStream;
/**
* Created by qianzhou on 5/6/15.
*/
-public class HadoopFileSystem implements FileSystem {
+class HadoopFileSystem implements FileSystem {
private static final Logger logger = LoggerFactory.getLogger(HadoopFileSystem.class);
final org.apache.hadoop.fs.FileSystem fileSystem;
- public HadoopFileSystem() {
+ HadoopFileSystem() {
try {
fileSystem = org.apache.hadoop.fs.FileSystem.get(HadoopUtil.getCurrentConfiguration());
} catch (IOException e) {
@@ -47,6 +47,16 @@ public class HadoopFileSystem implements FileSystem {
}
@Override
+ public void deleteOnExit(String path) {
+ try {
+ fileSystem.deleteOnExit(new Path(path));
+ } catch (IOException e) {
+ logger.error("error deleteOnExit, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
public boolean createDirectory(String path) {
try {
return fileSystem.mkdirs(new Path(path));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9e58ae87/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
index 1c14e3f..d512552 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
@@ -11,6 +11,9 @@ import java.io.*;
class LocalFileSystem implements FileSystem {
private static Logger logger = LoggerFactory.getLogger(LocalFileSystem.class);
+
+ LocalFileSystem(){}
+
@Override
public boolean checkExistence(String path) {
return new File(path).exists();
@@ -22,6 +25,11 @@ class LocalFileSystem implements FileSystem {
}
@Override
+ public void deleteOnExit(String path) {
+ new File(path).deleteOnExit();
+ }
+
+ @Override
public boolean createDirectory(String path) {
return new File(path).mkdirs();
}
@@ -42,7 +50,7 @@ class LocalFileSystem implements FileSystem {
return new FileOutputStream(path);
} catch (FileNotFoundException e) {
//should not happen
- logger.error("path:" + path + " nout found");
+ logger.error("path:" + path + " out found");
throw new RuntimeException(e);
}
}
@@ -53,7 +61,7 @@ class LocalFileSystem implements FileSystem {
return new FileInputStream(path);
} catch (FileNotFoundException e) {
//should not happen
- logger.error("path:" + path + " nout found");
+ logger.error("path:" + path + " out found");
throw new RuntimeException(e);
}
}
[31/50] [abbrv] incubator-kylin git commit: KYLIN-728
Posted by sh...@apache.org.
KYLIN-728
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0d87e8f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0d87e8f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0d87e8f6
Branch: refs/heads/streaming-localdict
Commit: 0d87e8f689e42d5a21330815eca6b2f83afb3f02
Parents: 876ac60
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue May 12 13:59:54 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue May 12 13:59:54 2015 +0800
----------------------------------------------------------------------
.../kylin/common/util/BytesSerializer.java | 6 +-
.../org/apache/kylin/common/util/SSHClient.java | 5 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 14 +-
.../apache/kylin/dict/DictionarySerializer.java | 54 +++++
.../invertedindex/model/IIKeyValueCodec.java | 31 +--
.../job/hadoop/cubev2/InMemCubeBuilder.java | 197 +++++++++++++------
.../kylin/job/hadoop/cubev2/InMemCuboidJob.java | 2 +-
.../hadoop/cubev2/MapContextGTRecordWriter.java | 2 +-
.../metadata/serializer/DataTypeSerializer.java | 32 +--
.../kylin/storage/cube/CubeCodeSystem.java | 1 +
.../storage/cube/CubeHBaseReadonlyStore.java | 14 +-
.../kylin/storage/gridtable/GTBuilder.java | 1 -
.../kylin/storage/gridtable/GTComboStore.java | 112 +++++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 10 +-
.../storage/gridtable/GTInvertedIndex.java | 4 +-
.../kylin/storage/gridtable/GTRawScanner.java | 24 +--
.../kylin/storage/gridtable/GTRecord.java | 4 +-
.../kylin/storage/gridtable/GTRowBlock.java | 19 +-
.../storage/gridtable/GTSampleCodeSystem.java | 5 +-
.../kylin/storage/gridtable/IGTScanner.java | 6 +-
.../kylin/storage/gridtable/IGTStore.java | 18 +-
.../apache/kylin/storage/gridtable/ScanKey.java | 34 ++++
.../storage/gridtable/diskstore/FileSystem.java | 22 +++
.../gridtable/diskstore/GTDiskStore.java | 160 +++++++++++++++
.../gridtable/diskstore/HadoopFileSystem.java | 88 +++++++++
.../gridtable/diskstore/LocalFileSystem.java | 60 ++++++
.../gridtable/memstore/GTSimpleMemStore.java | 49 +++--
.../apache/kylin/storage/util/SizeOfUtil.java | 21 ++
.../invertedindex/IIStreamBuilder.java | 6 +-
29 files changed, 803 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java b/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
index 1d1f5ae..26342f5 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
@@ -26,10 +26,10 @@ import java.nio.ByteBuffer;
*/
public interface BytesSerializer<T> {
- public static final int SERIALIZE_BUFFER_SIZE = 65536;
+ int SERIALIZE_BUFFER_SIZE = 65536;
- abstract public void serialize(T value, ByteBuffer out);
+ void serialize(T value, ByteBuffer out);
- abstract public T deserialize(ByteBuffer in);
+ T deserialize(ByteBuffer in);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
index ccaabf0..32eb72a 100644
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
@@ -49,6 +49,7 @@ public class SSHClient {
public SSHClient(String hostname, int port, String username, String password) {
this.hostname = hostname;
this.username = username;
+ this.port = port;
if (password != null && new File(password).exists()) {
this.identityPath = new File(password).getAbsolutePath();
this.password = null;
@@ -324,7 +325,7 @@ public class SSHClient {
throw ee;
}
if (timeout < 0)
- throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds.");
+ throw new Exception("Remote command not finished within " + timeoutSeconds + " seconds.");
}
channel.disconnect();
session.disconnect();
@@ -340,7 +341,7 @@ public class SSHClient {
jsch.addIdentity(identityPath);
}
- Session session = jsch.getSession(username, hostname, 22);
+ Session session = jsch.getSession(username, hostname, port);
if (password != null) {
session.setPassword(password);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 6f64116..07be092 100644
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -22,11 +22,7 @@ package org.apache.kylin.cube.cuboid;
* @author George Song (ysong1)
*
*/
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.cube.model.CubeDesc;
@@ -38,21 +34,21 @@ public class CuboidScheduler {
private final CubeDesc cubeDef;
private final int size;
private final long max;
- private final Map<Long, Collection<Long>> cache;
+ private final Map<Long, List<Long>> cache;
public CuboidScheduler(CubeDesc cube) {
this.cubeDef = cube;
this.size = cube.getRowkey().getRowKeyColumns().length;
this.max = (long) Math.pow(2, size) - 1;
- this.cache = new ConcurrentHashMap<Long, Collection<Long>>();
+ this.cache = new ConcurrentHashMap<Long, List<Long>>();
}
- public Collection<Long> getSpanningCuboid(long cuboid) {
+ public List<Long> getSpanningCuboid(long cuboid) {
if (cuboid > max || cuboid < 0) {
throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
}
- Collection<Long> result = cache.get(cuboid);
+ List<Long> result = cache.get(cuboid);
if (result != null) {
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
new file mode 100644
index 0000000..4b61591
--- /dev/null
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -0,0 +1,54 @@
+package org.apache.kylin.dict;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.common.util.ClassUtil;
+
+import java.io.*;
+
+/**
+ * Created by qianzhou on 5/5/15.
+ */
+public final class DictionarySerializer {
+
+ private DictionarySerializer() {}
+
+ public static Dictionary<?> deserialize(InputStream inputStream) {
+ try {
+ final DataInputStream dataInputStream = new DataInputStream(inputStream);
+ final String type = dataInputStream.readUTF();
+ final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+ dictionary.readFields(dataInputStream);
+ return dictionary;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
+ return deserialize(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
+ }
+
+ public static void serialize(Dictionary<?> dict, OutputStream outputStream) {
+ try {
+ DataOutputStream out = new DataOutputStream(outputStream);
+ out.writeUTF(dict.getClass().getName());
+ dict.write(out);
+ out.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static ImmutableBytesWritable serialize(Dictionary<?> dict) {
+ try {
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeUTF(dict.getClass().getName());
+ dict.write(out);
+ return new ImmutableBytesWritable(baos.toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 8dbaed7..b236879 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -23,8 +23,8 @@ import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionarySerializer;
import org.apache.kylin.invertedindex.index.ColumnValueContainer;
import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
@@ -32,7 +32,6 @@ import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
-import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
@@ -73,31 +72,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
if (dictionary == null) {
return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY));
} else {
- return new IIRow(key, value, serialize(dictionary));
- }
- }
-
- private static Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
- try {
- final DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
- final String type = dataInputStream.readUTF();
- final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
- dictionary.readFields(dataInputStream);
- return dictionary;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- private static ImmutableBytesWritable serialize(Dictionary<?> dict) {
- try {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos);
- out.writeUTF(dict.getClass().getName());
- dict.write(out);
- return new ImmutableBytesWritable(baos.toByteArray());
- } catch (IOException e) {
- throw new RuntimeException(e);
+ return new IIRow(key, value, DictionarySerializer.serialize(dictionary));
}
}
@@ -221,7 +196,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else {
final ImmutableBytesWritable dictBytes = row.getDictionary();
if (dictBytes.getLength() != 0) {
- final Dictionary<?> dictionary = deserialize(dictBytes);
+ final Dictionary<?> dictionary = DictionarySerializer.deserialize(dictBytes);
CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0);
c.fromBytes(row.getValue());
valueContainers[curCol] = c;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index 29cdc9a..87ad2d3 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -38,6 +38,7 @@ import com.google.common.collect.Maps;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.util.StringUtils;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
@@ -51,10 +52,9 @@ import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.serializer.DataTypeSerializer;
import org.apache.kylin.storage.cube.CubeGridTable;
import org.apache.kylin.storage.gridtable.*;
-import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.storage.util.SizeOfUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +69,7 @@ import java.util.concurrent.BlockingQueue;
@SuppressWarnings("rawtypes")
public class InMemCubeBuilder implements Runnable {
+ private static final double AGGREGATION_CACHE_FACTOR = 3;
private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class);
private BlockingQueue<List<String>> queue;
@@ -78,14 +79,11 @@ public class InMemCubeBuilder implements Runnable {
private Map<TblColRef, Dictionary<?>> dictionaryMap = null;
private CubeJoinedFlatTableDesc intermediateTableDesc;
private MeasureCodec measureCodec;
- private int measureNumber;
private String[] metricsAggrFuncs = null;
private Map<Integer, Integer> dependentMeasures = null; // key: index of Measure which depends on another measure; value: index of Measure which is depended on;
public static final LongWritable ONE = new LongWritable(1l);
protected IGTRecordWriter gtRecordWriter;
- private GridTable baseCuboidGT;
- private DataTypeSerializer[] serializers;
/**
@@ -95,30 +93,31 @@ public class InMemCubeBuilder implements Runnable {
* @param gtRecordWriter
*/
public InMemCubeBuilder(BlockingQueue<List<String>> queue, CubeInstance cube, Map<TblColRef, Dictionary<?>> dictionaryMap, IGTRecordWriter gtRecordWriter) {
+ if (dictionaryMap == null || dictionaryMap.isEmpty()) {
+ throw new IllegalArgumentException();
+ }
this.queue = queue;
this.desc = cube.getDescriptor();
this.cuboidScheduler = new CuboidScheduler(desc);
this.dictionaryMap = dictionaryMap;
this.gtRecordWriter = gtRecordWriter;
- baseCuboidId = Cuboid.getBaseCuboidId(desc);
-
- intermediateTableDesc = new CubeJoinedFlatTableDesc(desc, null);
- measureCodec = new MeasureCodec(desc.getMeasures());
- measureNumber = desc.getMeasures().size();
-
- dependentMeasures = Maps.newHashMap();
+ this.baseCuboidId = Cuboid.getBaseCuboidId(desc);
+ this.intermediateTableDesc = new CubeJoinedFlatTableDesc(desc, null);
+ this.measureCodec = new MeasureCodec(desc.getMeasures());
- Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
+ Map<String, Integer> measureIndexMap = Maps.newHashMap();
List<String> metricsAggrFuncsList = Lists.newArrayList();
- for (int i = 0, n = desc.getMeasures().size(); i < n; i++) {
+ final int measureCount = desc.getMeasures().size();
+ for (int i = 0; i < measureCount; i++) {
MeasureDesc measureDesc = desc.getMeasures().get(i);
metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
measureIndexMap.put(desc.getMeasures().get(i).getName(), i);
}
- metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
+ this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
- for (int i = 0; i < measureNumber; i++) {
+ this.dependentMeasures = Maps.newHashMap();
+ for (int i = 0; i < measureCount; i++) {
String depMsrRef = desc.getMeasures().get(i).getDependentMeasureRef();
if (depMsrRef != null) {
int index = measureIndexMap.get(depMsrRef);
@@ -126,21 +125,19 @@ public class InMemCubeBuilder implements Runnable {
}
}
- if (dictionaryMap == null || dictionaryMap.isEmpty())
- throw new IllegalArgumentException();
}
- private GridTable newGridTableByCuboidID(long cuboidID) {
+ private GridTable newGridTableByCuboidID(long cuboidID, boolean memStore) {
GTInfo info = CubeGridTable.newGTInfo(desc, cuboidID, dictionaryMap);
- GTSimpleMemStore store = new GTSimpleMemStore(info);
+ GTComboStore store = new GTComboStore(info, memStore);
GridTable gridTable = new GridTable(info, store);
return gridTable;
}
- private GridTable aggregateCuboid(GridTable parentCuboid, long parentCuboidId, long cuboidId) throws IOException {
+ private GridTable aggregateCuboid(GridTable parentCuboid, long parentCuboidId, long cuboidId, boolean inMem) throws IOException {
logger.info("Calculating cuboid " + cuboidId + " from parent " + parentCuboidId);
- Pair<BitSet, BitSet> columnBitSets = getDimensionAndMetricColumBitSet(parentCuboidId);
+ Pair<BitSet, BitSet> columnBitSets = getDimensionAndMetricColumnBitSet(parentCuboidId);
BitSet parentDimensions = columnBitSets.getFirst();
BitSet measureColumns = columnBitSets.getSecond();
BitSet childDimensions = (BitSet) parentDimensions.clone();
@@ -160,14 +157,14 @@ public class InMemCubeBuilder implements Runnable {
mask = mask >> 1;
}
- return scanAndAggregateGridTable(parentCuboid, cuboidId, childDimensions, measureColumns);
+ return scanAndAggregateGridTable(parentCuboid, cuboidId, childDimensions, measureColumns, inMem);
}
- private GridTable scanAndAggregateGridTable(GridTable gridTable, long cuboidId, BitSet aggregationColumns, BitSet measureColumns) throws IOException {
+ private GridTable scanAndAggregateGridTable(GridTable gridTable, long cuboidId, BitSet aggregationColumns, BitSet measureColumns, boolean inMem) throws IOException {
GTScanRequest req = new GTScanRequest(gridTable.getInfo(), null, aggregationColumns, measureColumns, metricsAggrFuncs, null);
IGTScanner scanner = gridTable.scan(req);
- GridTable newGridTable = newGridTableByCuboidID(cuboidId);
+ GridTable newGridTable = newGridTableByCuboidID(cuboidId, inMem);
GTBuilder builder = newGridTable.rebuild();
BitSet allNeededColumns = new BitSet();
@@ -181,7 +178,7 @@ public class InMemCubeBuilder implements Runnable {
try {
BitSet dependentMetrics = new BitSet(allNeededColumns.cardinality());
for (Integer i : dependentMeasures.keySet()) {
- dependentMetrics.set((allNeededColumns.cardinality() - measureNumber + dependentMeasures.get(i)));
+ dependentMetrics.set((allNeededColumns.cardinality() - desc.getMeasures().size() + dependentMeasures.get(i)));
}
Object[] hllObjects = new Object[dependentMeasures.keySet().size()];
@@ -197,13 +194,13 @@ public class InMemCubeBuilder implements Runnable {
for (Integer i : dependentMeasures.keySet()) {
for (int index = 0, c = dependentMetrics.nextSetBit(0); c >= 0; index++, c = dependentMetrics.nextSetBit(c + 1)) {
- if (c == allNeededColumns.cardinality() - measureNumber + dependentMeasures.get(i)) {
+ if (c == allNeededColumns.cardinality() - desc.getMeasures().size() + dependentMeasures.get(i)) {
assert hllObjects[index] instanceof HyperLogLogPlusCounter; // currently only HLL is allowed
byteBuffer.clear();
BytesUtil.writeVLong(((HyperLogLogPlusCounter) hllObjects[index]).getCountEstimate(), byteBuffer);
byteArray.set(byteBuffer.array(), 0, byteBuffer.position());
- newRecord.set(allNeededColumns.cardinality() - measureNumber + i, byteArray);
+ newRecord.set(allNeededColumns.cardinality() - desc.getMeasures().size() + i, byteArray);
}
}
@@ -219,16 +216,16 @@ public class InMemCubeBuilder implements Runnable {
return newGridTable;
}
- private Pair<BitSet, BitSet> getDimensionAndMetricColumBitSet(long cuboidId) {
+ private Pair<BitSet, BitSet> getDimensionAndMetricColumnBitSet(long cuboidId) {
BitSet bitSet = BitSet.valueOf(new long[]{cuboidId});
BitSet dimension = new BitSet();
dimension.set(0, bitSet.cardinality());
BitSet metrics = new BitSet();
- metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.measureNumber);
+ metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.desc.getMeasures().size());
return new Pair<BitSet, BitSet>(dimension, metrics);
}
- private Object[] buildKey(List<String> row, DataTypeSerializer[] serializers) {
+ private Object[] buildKey(List<String> row) {
int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
Object[] key = new Object[keySize];
@@ -280,7 +277,8 @@ public class InMemCubeBuilder implements Runnable {
@Override
public void run() {
try {
- createBaseCuboidGT();
+ logger.info("Create base cuboid " + baseCuboidId);
+ final GridTable baseCuboidGT = newGridTableByCuboidID(baseCuboidId, true);
GTBuilder baseGTBuilder = baseCuboidGT.rebuild();
final GTRecord baseGTRecord = new GTRecord(baseCuboidGT.getInfo());
@@ -339,7 +337,7 @@ public class InMemCubeBuilder implements Runnable {
}
};
- Pair<BitSet, BitSet> dimensionMetricsBitSet = getDimensionAndMetricColumBitSet(baseCuboidId);
+ Pair<BitSet, BitSet> dimensionMetricsBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId);
GTScanRequest req = new GTScanRequest(baseCuboidGT.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
IGTScanner aggregationScanner = new GTAggregateScanner(queueScanner, req);
@@ -352,8 +350,18 @@ public class InMemCubeBuilder implements Runnable {
aggregationScanner.close();
logger.info("Base cuboid has " + counter + " rows;");
- if (counter > 0)
- createNDCuboidGT(null, -1l, baseCuboidId);
+ SimpleGridTableTree tree = new SimpleGridTableTree();
+ tree.data = baseCuboidGT;
+ tree.id = baseCuboidId;
+ tree.parent = null;
+ if (counter > 0) {
+ List<Long> children = cuboidScheduler.getSpanningCuboid(baseCuboidId);
+ Collections.sort(children);
+ for (Long childId : children) {
+ createNDCuboidGT(tree, baseCuboidId, childId);
+ }
+ }
+ baseCuboidGT.getStore().drop();
} catch (IOException e) {
logger.error("Fail to build cube", e);
@@ -364,7 +372,7 @@ public class InMemCubeBuilder implements Runnable {
private void buildGTRecord(List<String> row, GTRecord record) {
- Object[] dimensions = buildKey(row, serializers);
+ Object[] dimensions = buildKey(row);
Object[] metricsValues = buildValue(row);
Object[] recordValues = new Object[dimensions.length + metricsValues.length];
System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length);
@@ -372,43 +380,84 @@ public class InMemCubeBuilder implements Runnable {
record.setValues(recordValues);
}
- private void createBaseCuboidGT() throws IOException {
-
- logger.info("Create base cuboid " + baseCuboidId);
- Cuboid baseCuboid = Cuboid.findById(this.desc, baseCuboidId);
- serializers = new DataTypeSerializer[baseCuboid.getColumns().size()];
-
- for (int i = 0; i < baseCuboid.getColumns().size(); i++) {
- serializers[i] = DataTypeSerializer.create(baseCuboid.getColumns().get(i).getType());
+ private boolean checkMemory(long threshold) {
+ final long freeMemory = Runtime.getRuntime().freeMemory();
+ logger.info("available memory:" + (freeMemory>>10) + " KB");
+ if (freeMemory >= threshold) {
+ logger.info("no need to flush to disk");
+ return true;
+ } else {
+ return false;
}
-
- this.baseCuboidGT = newGridTableByCuboidID(baseCuboidId);
}
+ private boolean gc(TreeNode<GridTable> parentNode) {
+ final long parentCuboidMem = SizeOfUtil.deepSizeOf(parentNode.data.getStore());
+ long threshold = (long) (parentCuboidMem * (AGGREGATION_CACHE_FACTOR + 1));
+ logger.info((threshold >> 10) + " KB is needed to create " + parentNode.id + "'s child");
+ if (checkMemory(threshold)) {
+ return true;
+ }
+ final List<TreeNode<GridTable>> gridTables = parentNode.getAncestorList();
+ for (TreeNode<GridTable> gridTable : gridTables) {
+ logger.info("wait 10 seconds for gc");
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ logger.error("this should not happen", e);
+ }
+ if (checkMemory(threshold)) {
+ return true;
+ } else {
+ logger.info("memory is low, try to select one node to flush to disk from:" + StringUtils.join(",", gridTables));
+ final IGTStore store = gridTable.data.getStore();
+ assert store instanceof GTComboStore;
+ if (store.memoryUsage() > 0) {
+ logger.info("cuboid id:" + gridTable.id + " selected, memory used:" + (SizeOfUtil.deepSizeOf(store)>>10) + " KB");
+ long t = System.currentTimeMillis();
+ ((GTComboStore) store).switchToDiskStore();
+ logger.info("switch to disk store cost:" + (System.currentTimeMillis() - t) + "ms");
+ }
+ }
+ }
+ logger.info("no store has been flushed to disk");
+ return true;
+ }
- private void createNDCuboidGT(GridTable parentCuboid, long parentCuboidId, long cuboidId) throws IOException {
+ private void createNDCuboidGT(SimpleGridTableTree parentNode, long parentCuboidId, long cuboidId) throws IOException {
- GridTable thisCuboid;
long startTime = System.currentTimeMillis();
- if (parentCuboidId < 0) {
- thisCuboid = this.baseCuboidGT;
- } else {
- thisCuboid = aggregateCuboid(parentCuboid, parentCuboidId, cuboidId);
+ assert parentNode.data.getStore() instanceof GTComboStore;
+ if (parentNode.data.getStore().memoryUsage() <= 0) {
+ long t = System.currentTimeMillis();
+ ((GTComboStore) parentNode.data.getStore()).switchToMemStore();
+ logger.info("switch to mem store cost:" + (System.currentTimeMillis() - t) + "ms");
}
- logger.info("Cuboid " + cuboidId + " build takes (second): " + (System.currentTimeMillis() - startTime) / 1000);
-
- ArrayList<Long> children = (ArrayList<Long>) cuboidScheduler.getSpanningCuboid(cuboidId);
- Collections.sort(children); // sort cuboids
- for (Long childId : children) {
- createNDCuboidGT(thisCuboid, cuboidId, childId);
+ boolean inMem = gc(parentNode);
+ GridTable currentCuboid = aggregateCuboid(parentNode.data, parentCuboidId, cuboidId, inMem);
+ SimpleGridTableTree node = new SimpleGridTableTree();
+ node.parent = parentNode;
+ node.data = currentCuboid;
+ node.id = cuboidId;
+ parentNode.children.add(node);
+
+ logger.info("Cuboid " + cuboidId + " build takes " + (System.currentTimeMillis() - startTime) + "ms");
+
+ List<Long> children = cuboidScheduler.getSpanningCuboid(cuboidId);
+ if (!children.isEmpty()) {
+ Collections.sort(children); // sort cuboids
+ for (Long childId : children) {
+ createNDCuboidGT(node, cuboidId, childId);
+ }
}
-
startTime = System.currentTimeMillis();
//output the grid table
- outputGT(cuboidId, thisCuboid);
- logger.info("Cuboid" + cuboidId + " output takes (second) " + (System.currentTimeMillis() - startTime) / 1000);
+ outputGT(cuboidId, currentCuboid);
+ currentCuboid.getStore().drop();
+ parentNode.children.remove(node);
+ logger.info("Cuboid" + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms");
}
@@ -419,4 +468,30 @@ public class InMemCubeBuilder implements Runnable {
this.gtRecordWriter.write(cuboidId, record);
}
}
+
+ private static class TreeNode<T> {
+ T data;
+ long id;
+ TreeNode<T> parent;
+ List<TreeNode<T>> children = Lists.newArrayList();
+
+ List<TreeNode<T>> getAncestorList() {
+ ArrayList<TreeNode<T>> result = Lists.newArrayList();
+ TreeNode<T> parent = this.parent;
+ while (parent != null) {
+ result.add(parent);
+ parent = parent.parent;
+ }
+ return Lists.reverse(result);
+ }
+
+ @Override
+ public String toString() {
+ return id + "";
+ }
+ }
+
+ private static class SimpleGridTableTree extends TreeNode<GridTable> {}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidJob.java
index fc165ea..db690b9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidJob.java
@@ -104,7 +104,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
// set job configuration
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- long timeout = 1000*60*60l; // 1 hour
+ long timeout = 1000*60*60L; // 1 hour
job.getConfiguration().set("mapred.task.timeout", String.valueOf(timeout));
Configuration conf = HBaseConfiguration.create(getConf());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
index 41237d7..3ba80d1 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
@@ -72,7 +72,7 @@ public class MapContextGTRecordWriter implements IGTRecordWriter {
try {
mapContext.write(outputKey, outputValue);
} catch (InterruptedException e) {
- throw new IOException(e);
+ throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
index ac6f409..739cde4 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/serializer/DataTypeSerializer.java
@@ -20,8 +20,11 @@ package org.apache.kylin.metadata.serializer;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.Map;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.metadata.model.DataType;
@@ -31,20 +34,23 @@ import org.apache.kylin.metadata.model.DataType;
*/
abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
- final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
+ final static Map<String, Class<?>> implementations;
static {
- implementations.put("varchar", StringSerializer.class);
- implementations.put("decimal", BigDecimalSerializer.class);
- implementations.put("double", DoubleSerializer.class);
- implementations.put("float", DoubleSerializer.class);
- implementations.put("bigint", LongSerializer.class);
- implementations.put("long", LongSerializer.class);
- implementations.put("integer", LongSerializer.class);
- implementations.put("int", LongSerializer.class);
- implementations.put("smallint", LongSerializer.class);
- implementations.put("date", DateTimeSerializer.class);
- implementations.put("datetime", DateTimeSerializer.class);
- implementations.put("timestamp", DateTimeSerializer.class);
+ HashMap<String, Class<?>> impl = Maps.newHashMap();
+ impl.put("varchar", StringSerializer.class);
+ impl.put("decimal", BigDecimalSerializer.class);
+ impl.put("double", DoubleSerializer.class);
+ impl.put("float", DoubleSerializer.class);
+ impl.put("bigint", LongSerializer.class);
+ impl.put("long", LongSerializer.class);
+ impl.put("integer", LongSerializer.class);
+ impl.put("int", LongSerializer.class);
+ impl.put("smallint", LongSerializer.class);
+ impl.put("date", DateTimeSerializer.class);
+ impl.put("datetime", DateTimeSerializer.class);
+ impl.put("timestamp", DateTimeSerializer.class);
+ implementations = Collections.unmodifiableMap(impl);
+
}
public static DataTypeSerializer<?> create(String dataType) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
index 70567e5..9b7550f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
@@ -8,6 +8,7 @@ import java.util.Map;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.dict.Dictionary;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index f721148..0784587 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -49,13 +49,8 @@ public class CubeHBaseReadonlyStore implements IGTStore {
}
@Override
- public GTInfo getInfo() {
- return info;
- }
-
- @Override
- public String getStorageDescription() {
- return cubeSeg.toString();
+ public long memoryUsage() {
+ return 0;
}
@Override
@@ -127,6 +122,11 @@ public class CubeHBaseReadonlyStore implements IGTStore {
};
}
+ @Override
+ public void drop() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
private Scan buildScan(ByteArray pkStart, ByteArray pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
Scan scan = new Scan();
scan.setCaching(SCAN_CACHE);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
index 7195e7f..7552ab3 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
@@ -51,7 +51,6 @@ public class GTBuilder implements Closeable, Flushable {
blockWriter.readyForFlush();
storeWriter.write(block);
writtenRowBlockCount++;
-
if (block.isFull()) {
blockWriter.clearForNext();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
new file mode 100644
index 0000000..c7d0c2b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -0,0 +1,112 @@
+package org.apache.kylin.storage.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.storage.gridtable.diskstore.GTDiskStore;
+import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+/**
+ * Created by qianzhou on 5/6/15.
+ */
+public class GTComboStore implements IGTStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(GTComboStore.class);
+
+ private final GTInfo gtInfo;
+
+ private void convert(IGTStore input, IGTStore output) throws IOException {
+ final IGTStoreScanner scanner = input.scan(ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), null, null);
+ final IGTStoreWriter writer = output.rebuild(-1);
+ while (scanner.hasNext()) {
+ writer.write(scanner.next());
+ }
+ }
+
+ private GTDiskStore gtDiskStore;
+ private GTSimpleMemStore gtSimpleMemStore;
+
+ public GTComboStore(GTInfo gtInfo) {
+ this(gtInfo, true);
+ }
+
+ public GTComboStore(GTInfo gtInfo, boolean useMemStore) {
+ this.gtInfo = gtInfo;
+ if (useMemStore) {
+ this.gtSimpleMemStore = new GTSimpleMemStore(gtInfo);
+ } else {
+ this.gtDiskStore = new GTDiskStore(gtInfo);
+ }
+ }
+
+ private IGTStore getCurrent() {
+ if (gtSimpleMemStore != null) {
+ return gtSimpleMemStore;
+ } else {
+ return gtDiskStore;
+ }
+ }
+
+ public void switchToMemStore() {
+ try {
+ if (gtSimpleMemStore == null) {
+ gtSimpleMemStore = new GTSimpleMemStore(gtInfo);
+ convert(gtDiskStore, gtSimpleMemStore);
+ gtDiskStore.drop();
+ gtDiskStore = null;
+ }
+ } catch (IOException e) {
+ logger.error("fail to switch to mem store", e);
+ throw new RuntimeException(e);
+ }
+ logger.info("switch to mem store");
+ }
+
+ public void switchToDiskStore() {
+ try {
+ if (gtDiskStore == null) {
+ gtDiskStore = new GTDiskStore(gtInfo);
+ convert(gtSimpleMemStore, gtDiskStore);
+ gtSimpleMemStore.drop();
+ gtSimpleMemStore = null;
+ }
+ } catch (IOException e) {
+ logger.error("fail to switch to disk store", e);
+ throw new RuntimeException(e);
+ }
+ logger.info("switch to disk store");
+ }
+
+ @Override
+ public long memoryUsage() {
+ return getCurrent().memoryUsage();
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) throws IOException {
+ return getCurrent().rebuild(shard);
+ }
+
+ @Override
+ public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException {
+ return getCurrent().append(shard, fillLast);
+ }
+
+ @Override
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ return getCurrent().scan(pkStart, pkEnd, selectedColBlocks, additionalPushDown);
+ }
+
+ @Override
+ public void drop() throws IOException {
+ if (gtSimpleMemStore != null) {
+ gtSimpleMemStore.drop();
+ }
+ if (gtDiskStore != null) {
+ gtDiskStore.drop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 954e464..fdabb60 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -17,7 +17,6 @@ public class GTInfo {
IGTCodeSystem codeSystem;
// column schema
- int nColumns;
DataType[] colTypes;
BitSet colAll;
BitSet colPreferIndex;
@@ -37,7 +36,7 @@ public class GTInfo {
}
public int getColumnCount() {
- return nColumns;
+ return colTypes.length;
}
public DataType getColumnType(int i) {
@@ -74,7 +73,7 @@ public class GTInfo {
public int getMaxColumnLength() {
int max = 0;
- for (int i = 0; i < nColumns; i++)
+ for (int i = 0; i < colTypes.length; i++)
max = Math.max(max, codeSystem.maxCodeLength(i));
return max;
}
@@ -95,7 +94,7 @@ public class GTInfo {
public TblColRef colRef(int i) {
if (colRefs == null) {
- colRefs = new TblColRef[nColumns];
+ colRefs = new TblColRef[colTypes.length];
}
if (colRefs[i] == null) {
colRefs[i] = GTUtil.tblColRef(i, colTypes[i].toString());
@@ -124,7 +123,7 @@ public class GTInfo {
private void validateColumnBlocks() {
colAll = new BitSet();
- colAll.flip(0, nColumns);
+ colAll.flip(0, colTypes.length);
if (colBlocks == null) {
colBlocks = new BitSet[2];
@@ -185,7 +184,6 @@ public class GTInfo {
/** required */
public Builder setColumns(DataType... colTypes) {
- info.nColumns = colTypes.length;
info.colTypes = colTypes;
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
index 7c6abec..2756659 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -34,7 +34,7 @@ public class GTInvertedIndex {
this.colPreferIndex = info.colPreferIndex;
this.colBlocks = info.selectColumnBlocks(colPreferIndex);
- index = new GTInvertedIndexOfColumn[info.nColumns];
+ index = new GTInvertedIndexOfColumn[info.getColumnCount()];
for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
index[i] = new GTInvertedIndexOfColumn(info.codeSystem.getFilterCodeSystem());
}
@@ -43,7 +43,7 @@ public class GTInvertedIndex {
public void add(GTRowBlock block) {
@SuppressWarnings("unchecked")
- Set<ByteArray>[] distinctValues = new Set[info.nColumns];
+ Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
distinctValues[i] = new HashSet<ByteArray>();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index 895ccf3..ff97cd5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -24,34 +24,14 @@ public class GTRawScanner implements IGTScanner {
public GTRawScanner(GTInfo info, IGTStore store, GTScanRequest req) throws IOException {
this.info = info;
- ByteArray start = makeScanKey(req.getPkStart());
- ByteArray end = makeScanKey(req.getPkEnd());
+ ByteArray start = ScanKey.makeScanKey(info, req.getPkStart());
+ ByteArray end = ScanKey.makeScanKey(info, req.getPkEnd());
this.selectedColBlocks = info.selectColumnBlocks(req.getColumns());
this.storeScanner = store.scan(start, end, selectedColBlocks, req);
this.oneRecord = new GTRecord(info);
}
- private ByteArray makeScanKey(GTRecord rec) {
- int firstPKCol = info.primaryKey.nextSetBit(0);
- if (rec == null || rec.cols[firstPKCol].array() == null)
- return null;
-
- BitSet selectedColumns = new BitSet();
- int len = 0;
- for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) {
- if (rec.cols[i].array() == null) {
- break;
- }
- selectedColumns.set(i);
- len += rec.cols[i].length();
- }
-
- ByteArray buf = ByteArray.allocate(len);
- rec.exportColumns(selectedColumns, buf);
- return buf;
- }
-
@Override
public GTInfo getInfo() {
return info;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 8516f05..2a38731 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -16,7 +16,7 @@ public class GTRecord implements Comparable<GTRecord> {
public GTRecord(GTInfo info) {
this.info = info;
- this.cols = new ByteArray[info.nColumns];
+ this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < cols.length; i++)
this.cols[i] = new ByteArray();
this.maskForEqualHashComp = info.colAll;
@@ -55,7 +55,7 @@ public class GTRecord implements Comparable<GTRecord> {
/** decode and return the values of this record */
public Object[] getValues() {
- return getValues(info.colAll, new Object[info.nColumns]);
+ return getValues(info.colAll, new Object[info.getColumnCount()]);
}
/** decode and return the values of this record */
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index 7f79948..ec24da6 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -1,5 +1,7 @@
package org.apache.kylin.storage.gridtable;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;
@@ -54,7 +56,7 @@ public class GTRowBlock {
public Writer getWriter() {
return new Writer();
}
-
+
public class Writer {
ByteBuffer[] cellBlockBuffers;
@@ -181,6 +183,21 @@ public class GTRowBlock {
return len;
}
+ public void export(DataOutputStream dataOutputStream) throws IOException {
+ dataOutputStream.writeInt(seqId);
+ dataOutputStream.writeInt(nRows);
+ export(dataOutputStream, primaryKey);
+ for (ByteArray cb : cellBlocks) {
+ export(dataOutputStream, cb);
+ }
+ }
+
+ public void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException {
+ dataOutputStream.writeInt(array.length());
+ dataOutputStream.write(array.array(), array.offset(), array.length());
+ }
+
+
/** write data to given buffer, like serialize */
public void export(ByteBuffer buf) {
buf.putInt(seqId);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index cb8698c..9c758fa 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -3,6 +3,7 @@ package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.measure.MeasureAggregator;
@@ -30,8 +31,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
public void init(GTInfo info) {
this.info = info;
- this.serializers = new DataTypeSerializer[info.nColumns];
- for (int i = 0; i < info.nColumns; i++) {
+ this.serializers = new DataTypeSerializer[info.getColumnCount()];
+ for (int i = 0; i < info.getColumnCount(); i++) {
this.serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
index 285a301..3d3c3c8 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
@@ -4,10 +4,10 @@ import java.io.Closeable;
public interface IGTScanner extends Iterable<GTRecord>, Closeable {
- public GTInfo getInfo();
+ GTInfo getInfo();
- public int getScannedRowCount();
+ int getScannedRowCount();
- public int getScannedRowBlockCount();
+ int getScannedRowBlockCount();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
index f5eb077..0152571 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -8,22 +8,22 @@ import java.util.Iterator;
import org.apache.kylin.common.util.ByteArray;
public interface IGTStore {
+
+ long memoryUsage();
- public GTInfo getInfo();
-
- public String getStorageDescription();
-
- public IGTStoreWriter rebuild(int shard) throws IOException;
+ IGTStoreWriter rebuild(int shard) throws IOException;
- public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
+ IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
- public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
+ IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
+
+ void drop() throws IOException;
- public interface IGTStoreWriter extends Closeable {
+ interface IGTStoreWriter extends Closeable {
void write(GTRowBlock block) throws IOException;
}
- public interface IGTStoreScanner extends Iterator<GTRowBlock>, Closeable {
+ interface IGTStoreScanner extends Iterator<GTRowBlock>, Closeable {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
new file mode 100644
index 0000000..5c0c436
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
@@ -0,0 +1,34 @@
+package org.apache.kylin.storage.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+
+import java.util.BitSet;
+
+/**
+ * Created by qianzhou on 5/6/15.
+ */
+public final class ScanKey {
+
+ private ScanKey() {
+ }
+
+ static ByteArray makeScanKey(GTInfo info, GTRecord rec) {
+ int firstPKCol = info.primaryKey.nextSetBit(0);
+ if (rec == null || rec.cols[firstPKCol].array() == null)
+ return null;
+
+ BitSet selectedColumns = new BitSet();
+ int len = 0;
+ for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) {
+ if (rec.cols[i].array() == null) {
+ break;
+ }
+ selectedColumns.set(i);
+ len += rec.cols[i].length();
+ }
+
+ ByteArray buf = ByteArray.allocate(len);
+ rec.exportColumns(selectedColumns, buf);
+ return buf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
new file mode 100644
index 0000000..2ab2c7e
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/FileSystem.java
@@ -0,0 +1,22 @@
+package org.apache.kylin.storage.gridtable.diskstore;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Created by qianzhou on 5/4/15.
+ */
+interface FileSystem {
+
+ boolean checkExistence(String path);
+
+ boolean delete(String path);
+
+ boolean createDirectory(String path);
+
+ boolean createFile(String path);
+
+ OutputStream getWriter(String path);
+
+ InputStream getReader(String path);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
new file mode 100644
index 0000000..f48fce3
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -0,0 +1,160 @@
+package org.apache.kylin.storage.gridtable.diskstore;
+
+import com.google.common.base.Preconditions;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRowBlock;
+import org.apache.kylin.storage.gridtable.GTScanRequest;
+import org.apache.kylin.storage.gridtable.IGTStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.UUID;
+
+/**
+ * Created by qianzhou on 5/4/15.
+ */
+public class GTDiskStore implements IGTStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(GTDiskStore.class);
+
+ private final String identifier;
+ private final FileSystem fileSystem;
+ private final DiskStoreWriter writer;
+ private final GTInfo gtInfo;
+
+ public GTDiskStore(GTInfo gtInfo) {
+ this.gtInfo = gtInfo;
+ this.fileSystem = new LocalFileSystem();
+ this.identifier = generateIdentifier(fileSystem);
+ logger.info("disk store created, identifier:" + identifier);
+ this.writer = new DiskStoreWriter(fileSystem.getWriter(getRowBlockFile(identifier)));
+ }
+
+ private String generateIdentifier(FileSystem fs) {
+ while (true) {
+ String identifier = UUID.randomUUID().toString();
+ final String path = getRootDirectory(identifier);
+ if (fs.createDirectory(path)) {
+ return identifier;
+ }
+ }
+ }
+
+ private String getRootDirectory(String identifier) {
+ return "/tmp/kylin/gtdiskstore/" + identifier;
+ }
+
+ private String getRowBlockFile(String identifier) {
+ return getRootDirectory(identifier) + "/rowblock";
+ }
+
+ private class DiskStoreWriter implements IGTStoreWriter {
+
+ private final DataOutputStream outputStream;
+
+ DiskStoreWriter(OutputStream outputStream) {
+ this.outputStream = new DataOutputStream(outputStream);
+ }
+
+ @Override
+ public void write(GTRowBlock block) throws IOException {
+ final int blockSize = block.exportLength();
+ outputStream.writeInt(blockSize);
+ block.export(outputStream);
+ outputStream.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ outputStream.close();
+ }
+ }
+
+ @Override
+ public long memoryUsage() {
+ return 0;
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) throws IOException {
+ return writer;
+ }
+
+ @Override
+ public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException {
+ return writer;
+ }
+
+ private class DiskStoreScanner implements IGTStoreScanner {
+
+ private final DataInputStream inputStream;
+ private int blockSize = 0;
+
+ DiskStoreScanner(InputStream inputStream) {
+ this.inputStream = new DataInputStream(inputStream);
+ }
+
+ @Override
+ public void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ try {
+ blockSize = inputStream.readInt();
+ return blockSize > 0;
+ } catch (EOFException e) {
+ return false;
+ } catch (IOException e) {
+ logger.error("input stream fail", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public GTRowBlock next() {
+ GTRowBlock block = new GTRowBlock(gtInfo);
+ ByteBuffer buffer = ByteBuffer.allocate(blockSize);
+ int count = blockSize;
+ while (count > 0) {
+ try {
+ count -= inputStream.read(buffer.array(), buffer.position(), buffer.remaining());
+ } catch (IOException e) {
+ logger.error("input stream fail", e);
+ throw new RuntimeException(e);
+ }
+ }
+ Preconditions.checkArgument(count == 0, "invalid read count:" + count + " block size:" + blockSize);
+ block.load(buffer);
+ return block;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ return new DiskStoreScanner(fileSystem.getReader(getRowBlockFile(identifier)));
+ }
+
+ @Override
+ public void drop() throws IOException {
+ try {
+ writer.close();
+ } catch (Exception e) {
+ logger.error("error to close writer", e);
+ }
+ fileSystem.delete(getRowBlockFile(identifier));
+ fileSystem.delete(getRootDirectory(identifier));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
new file mode 100644
index 0000000..e1efd1b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/HadoopFileSystem.java
@@ -0,0 +1,88 @@
+package org.apache.kylin.storage.gridtable.diskstore;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Created by qianzhou on 5/6/15.
+ */
+public class HadoopFileSystem implements FileSystem {
+
+ private static final Logger logger = LoggerFactory.getLogger(HadoopFileSystem.class);
+
+ final org.apache.hadoop.fs.FileSystem fileSystem;
+
+ public HadoopFileSystem() {
+ try {
+ fileSystem = org.apache.hadoop.fs.FileSystem.get(HadoopUtil.getCurrentConfiguration());
+ } catch (IOException e) {
+ logger.error("error construct HadoopFileSystem", e);
+ throw new RuntimeException(e);
+ }
+ }
+ @Override
+ public boolean checkExistence(String path) {
+ try {
+ return fileSystem.exists(new Path(path));
+ } catch (IOException e) {
+ logger.error("error checkExistence, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean delete(String path) {
+ try {
+ return fileSystem.delete(new Path(path), true);
+ } catch (IOException e) {
+ logger.error("error delete, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean createDirectory(String path) {
+ try {
+ return fileSystem.mkdirs(new Path(path));
+ } catch (IOException e) {
+ logger.error("error createDirectory, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean createFile(String path) {
+ try {
+ return fileSystem.createNewFile(new Path(path));
+ } catch (IOException e) {
+ logger.error("error createFile, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public OutputStream getWriter(String path) {
+ try {
+ return fileSystem.create(new Path(path));
+ } catch (IOException e) {
+ logger.error("error getWriter, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public InputStream getReader(String path) {
+ try {
+ return fileSystem.open(new Path(path));
+ } catch (IOException e) {
+ logger.error("error getReader, path:" + path, e);
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
new file mode 100644
index 0000000..1c14e3f
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/LocalFileSystem.java
@@ -0,0 +1,60 @@
+package org.apache.kylin.storage.gridtable.diskstore;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+
+/**
+ * Created by qianzhou on 5/4/15.
+ */
+class LocalFileSystem implements FileSystem {
+
+ private static Logger logger = LoggerFactory.getLogger(LocalFileSystem.class);
+ @Override
+ public boolean checkExistence(String path) {
+ return new File(path).exists();
+ }
+
+ @Override
+ public boolean delete(String path) {
+ return new File(path).delete();
+ }
+
+ @Override
+ public boolean createDirectory(String path) {
+ return new File(path).mkdirs();
+ }
+
+ @Override
+ public boolean createFile(String path) {
+ try {
+ return new File(path).createNewFile();
+ } catch (IOException e) {
+ logger.warn("create file failed:" + path, e);
+ return false;
+ }
+ }
+
+ @Override
+ public OutputStream getWriter(String path) {
+ try {
+ return new FileOutputStream(path);
+ } catch (FileNotFoundException e) {
+ //should not happen
+ logger.error("path:" + path + " nout found");
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public InputStream getReader(String path) {
+ try {
+ return new FileInputStream(path);
+ } catch (FileNotFoundException e) {
+ //should not happen
+ logger.error("path:" + path + " nout found");
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
index 32c7f36..329c048 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -1,44 +1,41 @@
package org.apache.kylin.storage.gridtable.memstore;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Iterator;
-import java.util.List;
-
+import com.google.common.collect.Lists;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.storage.gridtable.GTInfo;
import org.apache.kylin.storage.gridtable.GTRowBlock;
import org.apache.kylin.storage.gridtable.GTScanRequest;
import org.apache.kylin.storage.gridtable.IGTStore;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.List;
+
public class GTSimpleMemStore implements IGTStore {
- final GTInfo info;
final List<GTRowBlock> rowBlockList;
public GTSimpleMemStore(GTInfo info) {
- this.info = info;
- this.rowBlockList = new ArrayList<GTRowBlock>();
+ this.rowBlockList = Lists.newLinkedList();
if (info.isShardingEnabled())
throw new UnsupportedOperationException();
}
@Override
- public GTInfo getInfo() {
- return info;
- }
-
- @Override
- public String getStorageDescription() {
- return this.toString();
+ public long memoryUsage() {
+ if (rowBlockList.size() == 0) {
+ return 0;
+ } else {
+ return rowBlockList.get(0).exportLength() * Long.valueOf(rowBlockList.size());
+ }
}
@Override
public IGTStoreWriter rebuild(int shard) {
rowBlockList.clear();
- return new Writer();
+ return new Writer(rowBlockList);
}
@Override
@@ -47,10 +44,16 @@ public class GTSimpleMemStore implements IGTStore {
GTRowBlock last = rowBlockList.get(rowBlockList.size() - 1);
fillLast.copyFrom(last);
}
- return new Writer();
+ return new Writer(rowBlockList);
}
- private class Writer implements IGTStoreWriter {
+ private static class Writer implements IGTStoreWriter {
+
+ private final List<GTRowBlock> rowBlockList;
+
+ Writer(List<GTRowBlock> rowBlockList) {
+ this.rowBlockList = rowBlockList;
+ }
@Override
public void close() throws IOException {
}
@@ -66,7 +69,7 @@ public class GTSimpleMemStore implements IGTStore {
rowBlockList.add(copy);
}
}
- };
+ }
@Override
public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) {
@@ -95,4 +98,10 @@ public class GTSimpleMemStore implements IGTStore {
};
}
+ @Override
+ public void drop() throws IOException {
+ //will there be any concurrent issue? If yes, ArrayList should be replaced
+ rowBlockList.clear();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java b/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
new file mode 100644
index 0000000..d390415
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
@@ -0,0 +1,21 @@
+package org.apache.kylin.storage.util;
+
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+
+/**
+ * Created by qianzhou on 5/11/15.
+ */
+public final class SizeOfUtil {
+
+ private SizeOfUtil(){}
+
+ private static final ReflectionSizeOf DEFAULT_SIZE_OF = new ReflectionSizeOf();
+
+ public static final long deepSizeOf(Object obj) {
+ return DEFAULT_SIZE_OF.deepSizeOf(Integer.MAX_VALUE, true, obj).getCalculated();
+ }
+
+ public static final long sizeOf(Object obj) {
+ return DEFAULT_SIZE_OF.sizeOf(obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d87e8f6/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 773425b..7c1d435 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -71,10 +71,6 @@ public class IIStreamBuilder extends StreamBuilder {
private StreamingManager streamingManager;
public IIStreamBuilder(BlockingQueue<StreamMessage> queue, String streaming, String hTableName, IIDesc iiDesc, int shard) {
- this(queue, streaming, hTableName, iiDesc, shard, true);
- }
-
- public IIStreamBuilder(BlockingQueue<StreamMessage> queue, String streaming, String hTableName, IIDesc iiDesc, int shard, boolean useLocalDict) {
super(queue, iiDesc.getSliceSize());
this.streaming = streaming;
this.desc = iiDesc;
@@ -86,7 +82,7 @@ public class IIStreamBuilder extends StreamBuilder {
logger.error("cannot open htable name:" + hTableName, e);
throw new RuntimeException("cannot open htable name:" + hTableName, e);
}
- this.sliceBuilder = new SliceBuilder(desc, (short) shard, useLocalDict);
+ this.sliceBuilder = new SliceBuilder(desc, (short) shard, iiDesc.isUseLocalDictionary());
this.streamingManager = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv());
}
[33/50] [abbrv] incubator-kylin git commit: KYLIN-750 Merge cube
segments from HBase table, refine code
Posted by sh...@apache.org.
KYLIN-750 Merge cube segments from HBase table, refine code
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d1e4b9be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d1e4b9be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d1e4b9be
Branch: refs/heads/streaming-localdict
Commit: d1e4b9beb4efb298eb977df72e3a234e1e3adfab
Parents: 73c275e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue May 12 14:07:45 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue May 12 14:10:35 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/Array.java | 4 ---
.../java/org/apache/kylin/cube/CubeSegment.java | 22 ++++++++----
.../apache/kylin/job/cube/CubingJobBuilder.java | 1 -
.../job/hadoop/cube/FactDistinctColumnsJob.java | 7 ++--
.../hadoop/cube/FactDistinctColumnsReducer.java | 10 +++---
.../cube/FactDistinctHiveColumnsMapper.java | 2 +-
.../job/hadoop/cube/StorageCleanupJob.java | 37 +++++++++++---------
.../hadoop/cubev2/MapContextGTRecordWriter.java | 4 +--
.../hadoop/cubev2/MergeCuboidFromHBaseJob.java | 10 +++---
.../job/hadoop/cubev2/MergeStatisticsStep.java | 29 +++++++++------
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 23 ++++++------
11 files changed, 80 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/common/src/main/java/org/apache/kylin/common/util/Array.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Array.java b/common/src/main/java/org/apache/kylin/common/util/Array.java
index 7f64a65..c48bb3d 100644
--- a/common/src/main/java/org/apache/kylin/common/util/Array.java
+++ b/common/src/main/java/org/apache/kylin/common/util/Array.java
@@ -18,10 +18,6 @@
package org.apache.kylin.common.util;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import org.apache.commons.collections.PredicateUtils;
-
import java.util.Arrays;
/*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 382f5c2..fbc014b 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -18,23 +18,24 @@
package org.apache.kylin.cube;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Objects;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware {
@@ -335,5 +336,12 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware {
public void setSnapshots(ConcurrentHashMap<String, String> snapshots) {
this.snapshots = snapshots;
}
-
+
+ public String getStatisticsResourcePath() {
+ return getStatisticsResourcePath(this.getCubeInstance().getName(), this.getUuid());
+ }
+
+ public static String getStatisticsResourcePath(String cubeName, String cubeSegmentId) {
+ return ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeName + "/" + cubeSegmentId + ".seq";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index 6593ed2..e534441 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -443,7 +443,6 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg) + "/part-r-00000");
appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
appendExecCmdParameters(cmd, "statisticsenabled", String.valueOf(inMemoryCubing()));
- appendExecCmdParameters(cmd, "statisticssamplingpercent", String.valueOf(engineConfig.getConfig().getCubingInMemSamplingPercent()));
createHtableStep.setJobParams(cmd.toString());
createHtableStep.setJobClass(CreateHTableJob.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index bc10032..f8863a5 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.hadoop.cube;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -100,7 +99,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
int result = waitForCompletion(job);
- if(Boolean.parseBoolean(statistics_enabled)) {
+ if(result == 0 && Boolean.parseBoolean(statistics_enabled)) {
putStatisticsToResourceStore(statistics_output, newSegment);
}
@@ -145,14 +144,14 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
private void putStatisticsToResourceStore(String statisticsFolder, CubeSegment cubeSegment) throws IOException {
Path statisticsFilePath = new Path(statisticsFolder, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
- FileSystem fs = HadoopUtil.getFileSystem("hdfs:///" + statisticsFilePath);
+ FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
if (!fs.exists(statisticsFilePath))
throw new IOException("File " + statisticsFilePath + " does not exists;");
FSDataInputStream is = fs.open(statisticsFilePath);
try {
// put the statistics to metadata store
- String statisticsFileName = ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment.getUuid() + ".seq";
+ String statisticsFileName = cubeSegment.getStatisticsResourcePath();
ResourceStore rs = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index 5d29e1b..62bbd74 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -52,7 +52,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
private boolean collectStatistics = false;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
- // private Map<Long, Long> rowCountInCuboids;
protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
@@ -77,7 +76,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
if (collectStatistics) {
baseCuboidRowCountInMappers = Lists.newArrayList();
-// rowCountInCuboids = Maps.newHashMap();
cuboidHLLMap = Maps.newHashMap();
SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
}
@@ -113,7 +111,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
long cuboidId = 0 - key.get();
for (Text value : values) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(14);
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
@@ -139,7 +137,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
//output the hll info;
if (collectStatistics) {
writeMapperAndCuboidStatistics(context); // for human check
- writeCuboidStatistics(context.getConfiguration(), statisticsOutput, cuboidHLLMap); // for CreateHTableJob
+ writeCuboidStatistics(context.getConfiguration(), statisticsOutput, cuboidHLLMap, SAMPING_PERCENTAGE); // for CreateHTableJob
}
}
@@ -195,7 +193,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
}
- public static void writeCuboidStatistics(Configuration conf, String outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap) throws IOException {
+ public static void writeCuboidStatistics(Configuration conf, String outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
SequenceFile.Writer writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class),
@@ -205,6 +203,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
allCuboids.addAll(cuboidHLLMap.keySet());
Collections.sort(allCuboids);
+ // persist the sample percentage with key 0
+ writer.append(new LongWritable(0l), new BytesWritable(Bytes.toBytes(samplingPercentage)));
ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
try {
for (long i : allCuboids) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
index 0b75084..20df4c4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctHiveColumnsMapper.java
@@ -80,7 +80,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
allCuboidsHLL = new HyperLogLogPlusCounter[cuboidIds.length];
for (int i = 0; i < cuboidIds.length; i++) {
- allCuboidsHLL[i] = new HyperLogLogPlusCounter(16);
+ allCuboidsHLL[i] = new HyperLogLogPlusCounter(14);
}
hf = Hashing.md5();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
index 09fba01..2f80f0e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
@@ -18,18 +18,6 @@
package org.apache.kylin.job.hadoop.cube;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -43,6 +31,18 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,8 +118,10 @@ public class StorageCleanupJob extends AbstractHadoopJob {
for (CubeInstance cube : cubeMgr.listAllCubes()) {
for (CubeSegment seg : cube.getSegments()) {
String tablename = seg.getStorageLocationIdentifier();
- allTablesNeedToBeDropped.remove(tablename);
- log.info("Remove table " + tablename + " from drop list, as the table belongs to cube " + cube.getName() + " with status " + cube.getStatus());
+ if (allTablesNeedToBeDropped.contains(tablename)) {
+ allTablesNeedToBeDropped.remove(tablename);
+ log.info("Exclude table " + tablename + " from drop list, as the table belongs to cube " + cube.getName() + " with status " + cube.getStatus());
+ }
}
}
@@ -127,8 +129,11 @@ public class StorageCleanupJob extends AbstractHadoopJob {
for (IIInstance ii : iiManager.listAllIIs()) {
for (IISegment seg : ii.getSegments()) {
String tablename = seg.getStorageLocationIdentifier();
- allTablesNeedToBeDropped.remove(tablename);
- log.info("Remove table " + tablename + " from drop list, as the table belongs to ii " + ii.getName() + " with status " + ii.getStatus());
+
+ if (allTablesNeedToBeDropped.contains(tablename)) {
+ allTablesNeedToBeDropped.remove(tablename);
+ log.info("Exclude table " + tablename + " from drop list, as the table belongs to ii " + ii.getName() + " with status " + ii.getStatus());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
index 3ba80d1..df3f345 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
@@ -23,7 +23,7 @@ import java.util.BitSet;
public class MapContextGTRecordWriter implements IGTRecordWriter {
private static final Log logger = LogFactory.getLog(MapContextGTRecordWriter.class);
- protected MapContext mapContext;
+ protected MapContext<?, ?, ImmutableBytesWritable, Text> mapContext;
private Long lastCuboidId;
protected CubeSegment cubeSegment;
protected CubeDesc cubeDesc;
@@ -37,7 +37,7 @@ public class MapContextGTRecordWriter implements IGTRecordWriter {
private Text outputValue = new Text();
long cuboidRowCount = 0;
- public MapContextGTRecordWriter(MapContext mapContext, CubeDesc cubeDesc, CubeSegment cubeSegment) {
+ public MapContextGTRecordWriter(MapContext<?, ?, ImmutableBytesWritable, Text> mapContext, CubeDesc cubeDesc, CubeSegment cubeSegment) {
this.mapContext = mapContext;
this.cubeDesc = cubeDesc;
this.cubeSegment = cubeSegment;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
index 4f50279..b2172bc 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseJob.java
@@ -18,6 +18,9 @@
package org.apache.kylin.job.hadoop.cubev2;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -28,21 +31,16 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.cube.CuboidJob;
-import org.apache.kylin.job.hadoop.cube.CuboidReducer;
-
-import java.util.ArrayList;
-import java.util.List;
/**
* @author shaoshi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
index 5fe3f96..64ea14c 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -78,8 +79,9 @@ public class MergeStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
+ int averageSamplingPercentage = 0;
for (String segmentId : this.getMergingSegmentIds()) {
- String fileKey = ResourceStore.CUBE_STATISTICS_ROOT + "/" + getCubeName() + "/" + segmentId + ".seq";
+ String fileKey = CubeSegment.getStatisticsResourcePath(getCubeName(), segmentId);
InputStream is = rs.getResource(fileKey);
File tempFile = null;
FileOutputStream tempFileStream = null;
@@ -99,14 +101,19 @@ public class MergeStatisticsStep extends AbstractExecutable {
LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
- ByteArray byteArray = new ByteArray(value.getBytes());
- hll.readRegisters(byteArray.asBuffer());
-
- if (cuboidHLLMap.get(key.get()) != null) {
- cuboidHLLMap.get(key.get()).merge(hll);
+ if (key.get() == 0l) {
+ // sampling percentage;
+ averageSamplingPercentage += Bytes.toInt(value.getBytes());
} else {
- cuboidHLLMap.put(key.get(), hll);
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(14);
+ ByteArray byteArray = new ByteArray(value.getBytes());
+ hll.readRegisters(byteArray.asBuffer());
+
+ if (cuboidHLLMap.get(key.get()) != null) {
+ cuboidHLLMap.get(key.get()).merge(hll);
+ } else {
+ cuboidHLLMap.put(key.get(), hll);
+ }
}
}
} catch (Exception e) {
@@ -116,14 +123,14 @@ public class MergeStatisticsStep extends AbstractExecutable {
IOUtils.closeStream(reader);
}
}
-
- FactDistinctColumnsReducer.writeCuboidStatistics(conf, getMergedStatisticsPath(), cuboidHLLMap);
+ averageSamplingPercentage = averageSamplingPercentage / this.getMergingSegmentIds().size();
+ FactDistinctColumnsReducer.writeCuboidStatistics(conf, getMergedStatisticsPath(), cuboidHLLMap, averageSamplingPercentage);
Path statisticsFilePath = new Path(getMergedStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
FileSystem fs = statisticsFilePath.getFileSystem(conf);
FSDataInputStream is = fs.open(statisticsFilePath);
try {
// put the statistics to metadata store
- String statisticsFileName = ResourceStore.CUBE_STATISTICS_ROOT + "/" + getCubeName() + "/" + newSegment.getUuid() + ".seq";
+ String statisticsFileName = newSegment.getStatisticsResourcePath();
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
IOUtils.closeStream(is);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d1e4b9be/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index 6e884d5..8f802b7 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -81,7 +81,6 @@ public class CreateHTableJob extends AbstractHadoopJob {
CubeDesc cubeDesc = null;
String segmentName = null;
KylinConfig kylinConfig;
- private int SAMPING_PERCENTAGE = 5;
@Override
public int run(String[] args) throws Exception {
@@ -92,16 +91,11 @@ public class CreateHTableJob extends AbstractHadoopJob {
options.addOption(OPTION_PARTITION_FILE_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_STATISTICS_ENABLED);
- options.addOption(OPTION_STATISTICS_SAMPLING_PERCENT);
parseOptions(options, args);
Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
boolean statistics_enabled = Boolean.parseBoolean(getOptionValue(OPTION_STATISTICS_ENABLED));
- String statistics_sampling_percent = getOptionValue(OPTION_STATISTICS_SAMPLING_PERCENT);
-
- SAMPING_PERCENTAGE = Integer.parseInt(statistics_sampling_percent);
-
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
kylinConfig = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
@@ -228,7 +222,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
long totalSizeInM = 0;
ResourceStore rs = ResourceStore.getStore(kylinConfig);
- String fileKey = ResourceStore.CUBE_STATISTICS_ROOT + "/" + cube.getName() + "/" + cubeSegment.getUuid() + ".seq";
+ String fileKey = cubeSegment.getStatisticsResourcePath();
InputStream is = rs.getResource(fileKey);
File tempFile = null;
FileOutputStream tempFileStream = null;
@@ -241,18 +235,23 @@ public class CreateHTableJob extends AbstractHadoopJob {
IOUtils.closeStream(tempFileStream);
}
- FileSystem fs = HadoopUtil.getFileSystem("file:///" +tempFile.getAbsolutePath());
+ FileSystem fs = HadoopUtil.getFileSystem("file:///" + tempFile.getAbsolutePath());
SequenceFile.Reader reader = null;
try {
reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf);
LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ int samplingPercentage = 25;
while (reader.next(key, value)) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(16);
- ByteArray byteArray = new ByteArray(value.getBytes());
- hll.readRegisters(byteArray.asBuffer());
+ if (key.get() == 0l) {
+ samplingPercentage = Bytes.toInt(value.getBytes());
+ } else {
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(14);
+ ByteArray byteArray = new ByteArray(value.getBytes());
+ hll.readRegisters(byteArray.asBuffer());
- cuboidSizeMap.put(key.get(), hll.getCountEstimate() * 100 / SAMPING_PERCENTAGE);
+ cuboidSizeMap.put(key.get(), hll.getCountEstimate() * 100 / samplingPercentage);
+ }
}
} catch (Exception e) {
[03/50] [abbrv] incubator-kylin git commit: KYLIN-748
SqlHiveDataTypeMapping
Posted by sh...@apache.org.
KYLIN-748 SqlHiveDataTypeMapping
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5a19aae2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5a19aae2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5a19aae2
Branch: refs/heads/streaming-localdict
Commit: 5a19aae2b8ab723082825fa399152292f063d6c7
Parents: b376b60
Author: honma <ho...@ebay.com>
Authored: Mon May 4 18:23:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 4 18:23:03 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5a19aae2/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
index 0aec98f..f710d34 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
@@ -26,6 +26,8 @@ public class SqlHiveDataTypeMapping {
public static String getHiveDataType(String javaDataType) {
String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
+ hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType;
+
return hiveDataType.toLowerCase();
}
}
[14/50] [abbrv] incubator-kylin git commit: KYLIN-749 new test case
added, need manually insert data into kafka
Posted by sh...@apache.org.
KYLIN-749 new test case added, need manually insert data into kafka
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c8bc428a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c8bc428a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c8bc428a
Branch: refs/heads/streaming-localdict
Commit: c8bc428a93d8506d79ae008ccc5f7d4dbef5cc72
Parents: 23c61d2
Author: honma <ho...@ebay.com>
Authored: Tue May 5 16:41:47 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed May 6 10:16:48 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 6 +--
.../org/apache/kylin/common/util/JsonUtil.java | 9 ++--
.../org/apache/kylin/common/util/SortUtil.java | 21 ++++++++
.../invertedindex/test_streaming_table.json | 4 +-
.../test_streaming_table_model_desc.json | 1 +
.../streaming/test_streaming_table.json | 20 ++++++++
.../kylin/job/streaming/KafkaDataLoader.java | 46 +++++++++++++++++
.../java/org/apache/kylin/job/DataGenTest.java | 52 ++++++++++++++++++--
.../job/dataGen/StreamingDataGenerator.java | 36 ++++++++------
.../apache/kylin/metadata/model/ColumnDesc.java | 2 +-
.../kylin/streaming/JsonStreamParser.java | 8 +--
11 files changed, 169 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 7f12ac5..d184a24 100644
--- a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -15,7 +15,7 @@ public class DateFormat {
static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
- static SimpleDateFormat getDateFormat(String datePattern) {
+ public static SimpleDateFormat getDateFormat(String datePattern) {
ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
if (formatThreadLocal == null) {
threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
@@ -37,10 +37,6 @@ public class DateFormat {
return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis));
}
- public static String dateToString(Date date) {
- return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- }
-
public static String dateToString(Date date, String pattern) {
return getDateFormat(pattern).format(date);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 2a1e314..5b3a22c 100644
--- a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -18,12 +18,6 @@
package org.apache.kylin.common.util;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -32,12 +26,15 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import java.io.*;
+
public class JsonUtil {
// reuse the object mapper to save memory footprint
private static final ObjectMapper mapper = new ObjectMapper();
private static final ObjectMapper indentMapper = new ObjectMapper();
+
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
new file mode 100644
index 0000000..ac9f216
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
@@ -0,0 +1,21 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.base.Function;
+import com.google.common.collect.TreeMultimap;
+
+import java.util.Iterator;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class SortUtil {
+ public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
+ TreeMultimap<E, T> reorgnized = TreeMultimap.create();
+ while (input.hasNext()) {
+ T t = input.next();
+ E e = extractor.apply(t);
+ reorgnized.put(e, t);
+ }
+ return reorgnized.values().iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
index dc341aa..cbcca12 100644
--- a/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
+++ b/examples/test_case_data/localmeta/invertedindex/test_streaming_table.json
@@ -8,11 +8,11 @@
"segments": [
{
"uuid": null,
- "name": "19700101000000_20140901000000",
+ "name": "19700101000000_20190901000000",
"status": "READY",
"dictionaries": {
},
- "storage_location_identifier": "",
+ "storage_location_identifier": "KYLIN_2STEAMTEST",
"date_range_start": 0,
"date_range_end": 0,
"size_kb": 0,
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
index 66245ae..d0abb91 100644
--- a/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_streaming_table_model_desc.json
@@ -13,6 +13,7 @@
]
}
],
+ "lookups": [],
"metrics": [
"gmv",
"item_count"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/examples/test_case_data/localmeta/streaming/test_streaming_table.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/streaming/test_streaming_table.json b/examples/test_case_data/localmeta/streaming/test_streaming_table.json
new file mode 100644
index 0000000..537919a
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/test_streaming_table.json
@@ -0,0 +1,20 @@
+{
+ "uuid": "8b2b9dfe-900c-4d39-bf89-8472ec909322",
+ "name": "test_streaming_table",
+ "zookeeper": "sandbox:2181",
+ "topic": "test_streaming_table",
+ "timeout": 60000,
+ "maxReadCount": 1000,
+ "bufferSize": 65536,
+ "iiName": "test_streaming_table",
+ "parserName": "org.apache.kylin.streaming.JsonStreamParser",
+ "partition": 1,
+ "last_modified": 0,
+ "brokers": [
+ {
+ "id": 0,
+ "host": "sandbox",
+ "port": 6667
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
new file mode 100644
index 0000000..5577dce
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/streaming/KafkaDataLoader.java
@@ -0,0 +1,46 @@
+package org.apache.kylin.job.streaming;
+
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+import kafka.producer.ProducerConfig;
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.streaming.KafkaConfig;
+import org.apache.kylin.streaming.StreamingManager;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/5/15.
+ */
+public class KafkaDataLoader {
+ /**
+ *
+ * @param args args[0] data file path, args[1] streaming name
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ StreamingManager streamingManager = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KafkaConfig kafkaConfig = streamingManager.getKafkaConfig(args[1]);
+
+ List<String> alldata = FileUtils.readLines(new File(args[0]));
+
+ Properties props = new Properties();
+ props.put("metadata.broker.list", "sandbox:6667");
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("request.required.acks", "1");
+
+ ProducerConfig config = new ProducerConfig(props);
+
+ Producer<String, String> producer = new Producer<String, String>(config);
+
+ for (int i = 0; i < alldata.size(); ++i) {
+ KeyedMessage<String, String> data = new KeyedMessage<String, String>(kafkaConfig.getTopic(), String.valueOf(i), alldata.get(i));
+ producer.send(data);
+ }
+ producer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
index c05ddc7..4e3cf17 100644
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
@@ -18,15 +18,28 @@
package org.apache.kylin.job;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
+import com.fasterxml.jackson.databind.type.SimpleType;
+import com.google.common.base.Function;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.SortUtil;
import org.apache.kylin.job.dataGen.FactTableGenerator;
+import org.apache.kylin.job.dataGen.StreamingDataGenerator;
+import org.apache.kylin.metadata.MetadataManager;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
/**
* Created by honma on 6/19/14.
@@ -56,6 +69,39 @@ public class DataGenTest extends LocalFileMetadataTestCase {
@Test
public void testStreaming() throws Exception {
+ int totalCount = 10000;
+ int counter = 0;
+
+ Iterator<String> iterator = StreamingDataGenerator.generate(DateFormat.stringToMillis("2015-01-03"), DateFormat.stringToMillis("2015-01-06"), totalCount);
+
+ iterator = SortUtil.extractAndSort(iterator, new Function<String, Comparable>() {
+ public Comparable apply(String input) {
+ return getTsStr(input);
+ }
+ });
+
+ long lastTs = 0;
+ while (iterator.hasNext()) {
+ counter++;
+ String row = iterator.next();
+ System.out.println(row);
+ long ts = Long.parseLong(getTsStr(row));
+ Assert.assertTrue(ts >= lastTs);
+ lastTs = ts;
+ }
+ Assert.assertEquals(totalCount, counter);
}
+ final JavaType javaType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
+ final ObjectMapper objectMapper = new ObjectMapper();
+
+ private String getTsStr(String input) {
+ Map<String, String> json ;
+ try {
+ json = objectMapper.readValue(input.getBytes(), javaType);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return json.get("ts");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
index d822a9a..bf915e6 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/StreamingDataGenerator.java
@@ -1,17 +1,18 @@
package org.apache.kylin.job.dataGen;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -25,23 +26,23 @@ public class StreamingDataGenerator {
private static Random random = new Random();
private static String[] decimalFormat = new String[] { "%.4f", "%.5f", "%.6f" };
- public static Iterator<List<String>> generate(final long start, final long end, final int count) {
+ public static Iterator<String> generate(final long start, final long end, final int count) {
final KylinConfig config = KylinConfig.getInstanceFromEnv();
final IIInstance ii = IIManager.getInstance(config).getII("test_streaming_table");
final IIDesc iiDesc = ii.getDescriptor();
- final MetadataManager metadataManager = MetadataManager.getInstance(config);
- final ColumnDesc[] columnDescs = metadataManager.getTableDesc(iiDesc.getFactTableName()).getColumns();
+ final List<TblColRef> columns = iiDesc.listAllColumns();
- return new Iterator<List<String>>() {
- private Map<String, String> values = Maps.newHashMap();
+ return new Iterator<String>() {
+ private Map<String, String> values = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ private int index = 0;
@Override
public boolean hasNext() {
- return false;
+ return this.index < count;
}
@Override
- public List<String> next() {
+ public String next() {
values.clear();
long ts = this.createTs(start, end);
values.put("ts", Long.toString(ts));
@@ -53,16 +54,19 @@ public class StreamingDataGenerator {
values.put("gmv", String.format(decimalFormat[random.nextInt(3)], random.nextFloat() * 100));
values.put("item_count", Integer.toString(random.nextInt(5)));
- if (values.size() != columnDescs.length) {
+ if (values.size() != columns.size()) {
throw new RuntimeException("the structure of streaming table has changed, need to modify generator too");
}
- List<String> ret = Lists.newArrayList();
- for (ColumnDesc columnDesc : columnDescs) {
- String name = columnDesc.getName();
- ret.add(values.get(name));
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ try {
+ JsonUtil.writeValue(os, values);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
}
- return ret;
+ index++;
+ return new String(os.toByteArray());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 8d7ebfd..203d304 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -73,7 +73,7 @@ public class ColumnDesc {
}
public void setDatatype(String datatype) {
- logger.info("setting datatype to " + datatype);
+ //logger.info("setting datatype to " + datatype);
this.datatype = datatype;
type = DataType.getInstance(datatype);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c8bc428a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
index d762a14..d229647 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/JsonStreamParser.java
@@ -34,6 +34,7 @@
package org.apache.kylin.streaming;
+import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.SimpleType;
@@ -54,18 +55,19 @@ import java.util.Map;
public final class JsonStreamParser implements StreamParser {
private static final Logger logger = LoggerFactory.getLogger(JsonStreamParser.class);
+ private static final JavaType javaType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
+ private static final ObjectMapper objectMapper = new ObjectMapper();
private final List<TblColRef> allColumns;
- public JsonStreamParser(List<TblColRef> allColumns){
+ public JsonStreamParser(List<TblColRef> allColumns) {
this.allColumns = allColumns;
}
@Override
public List<String> parse(StreamMessage streamMessage) {
try {
- Map<String, String> json = new ObjectMapper().readValue(
- streamMessage.getRawData(), MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class)));
+ Map<String, String> json = objectMapper.readValue(streamMessage.getRawData(), javaType);
ArrayList<String> result = Lists.newArrayList();
for (TblColRef column : allColumns) {
for (Map.Entry<String, String> entry : json.entrySet()) {