You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/01/05 02:42:59 UTC
kylin git commit: KYLIN-2317 Support HybridCube
Repository: kylin
Updated Branches:
refs/heads/yang22 979ceebb9 -> 4f6515d90
KYLIN-2317 Support HybridCube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f6515d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f6515d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f6515d9
Branch: refs/heads/yang22
Commit: 4f6515d907396a08bd50c61eb5b72eda25afb6f0
Parents: 979ceeb
Author: Billy Liu <bi...@apache.org>
Authored: Thu Jan 5 10:42:51 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 5 10:42:51 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/cachesync/Broadcaster.java | 6 +-
.../kylin/storage/hybrid/HybridManager.java | 3 +-
.../localmeta/cube/ssb_cube1.json | 14 ++
.../localmeta/cube/ssb_cube2.json | 14 ++
.../localmeta/cube/ssb_cube3.json | 14 ++
.../localmeta/cube_desc/ssb_cube1.json | 155 ++++++++++++++
.../localmeta/cube_desc/ssb_cube2.json | 155 ++++++++++++++
.../localmeta/cube_desc/ssb_cube3.json | 143 +++++++++++++
.../kylin/rest/controller/BasicController.java | 8 +
.../kylin/rest/controller/HybridController.java | 84 ++++++++
.../kylin/rest/request/HybridRequest.java | 62 ++++++
.../kylin/rest/service/HybridService.java | 155 ++++++++++++++
.../apache/kylin/rest/service/ModelService.java | 4 +-
tool/pom.xml | 2 +
.../org/apache/kylin/tool/HybridCubeCLI.java | 206 +++++++++++++++++++
.../apache/kylin/tool/MrJobInfoExtractor.java | 25 +--
.../apache/kylin/tool/HybridCubeCLITest.java | 92 +++++++++
17 files changed, 1124 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 6bc5771..d00c490 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -115,7 +115,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
- final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size());
+ final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
@@ -127,7 +127,7 @@ public class Broadcaster {
try {
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(), broadcastEvent.getCacheKey());
} catch (IOException e) {
- logger.warn("Thread failed during wipe cache at " + broadcastEvent);
+ logger.warn("Thread failed during wipe cache at " + broadcastEvent, e);
}
}
});
@@ -228,7 +228,7 @@ public class Broadcaster {
try {
counter.incrementAndGet();
- broadcastEvents.putFirst(new BroadcastEvent(entity, event, key));
+ broadcastEvents.putLast(new BroadcastEvent(entity, event, key));
} catch (Exception e) {
counter.decrementAndGet();
logger.error("error putting BroadcastEvent", e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 4f81b09..748e873 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -124,10 +124,11 @@ public class HybridManager implements IRealizationProvider {
}
}
- private void reloadAllHybridInstance() throws IOException {
+ public void reloadAllHybridInstance() throws IOException {
ResourceStore store = getStore();
List<String> paths = store.collectResourceRecursively(ResourceStore.HYBRID_RESOURCE_ROOT, ".json");
+ hybridMap.clear();
logger.debug("Loading Hybrid from folder " + store.getReadableResourcePath(ResourceStore.HYBRID_RESOURCE_ROOT));
for (String path : paths) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube/ssb_cube1.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ssb_cube1.json b/examples/test_case_data/localmeta/cube/ssb_cube1.json
new file mode 100644
index 0000000..b1eb2a5
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb_cube1.json
@@ -0,0 +1,14 @@
+{
+ "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c70",
+ "name" : "ssb_cube1",
+ "owner" : "ADMIN",
+ "cost" : 50,
+ "status" : "DISABLED",
+ "segments" : [ ],
+ "last_modified" : 1457534216410,
+ "descriptor" : "ssb_cube1",
+ "create_time_utc" : 1457444500888,
+ "size_kb" : 0,
+ "input_records_count" : 0,
+ "input_records_size" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube/ssb_cube2.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ssb_cube2.json b/examples/test_case_data/localmeta/cube/ssb_cube2.json
new file mode 100644
index 0000000..58f935d
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb_cube2.json
@@ -0,0 +1,14 @@
+{
+ "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c71",
+ "name" : "ssb_cube2",
+ "owner" : "ADMIN",
+ "cost" : 50,
+ "status" : "DISABLED",
+ "segments" : [ ],
+ "last_modified" : 1457534216410,
+ "descriptor" : "ssb_cube2",
+ "create_time_utc" : 1457444500888,
+ "size_kb" : 0,
+ "input_records_count" : 0,
+ "input_records_size" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube/ssb_cube3.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/ssb_cube3.json b/examples/test_case_data/localmeta/cube/ssb_cube3.json
new file mode 100644
index 0000000..fc09006
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb_cube3.json
@@ -0,0 +1,14 @@
+{
+ "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c72",
+ "name" : "ssb_cube3",
+ "owner" : "ADMIN",
+ "cost" : 50,
+ "status" : "DISABLED",
+ "segments" : [ ],
+ "last_modified" : 1457534216410,
+ "descriptor" : "ssb_cube3",
+ "create_time_utc" : 1457444500888,
+ "size_kb" : 0,
+ "input_records_count" : 0,
+ "input_records_size" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
new file mode 100644
index 0000000..5986057
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
@@ -0,0 +1,155 @@
+{
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851058",
+ "name" : "ssb_cube1",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "C_CITY",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_CITY",
+ "derived" : null
+ }, {
+ "name" : "C_REGION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_REGION",
+ "derived" : null
+ }, {
+ "name" : "C_NATION",
+ "table" : "SSB.CUSTOMER",
+ "column" : "C_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_V_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "V_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "C_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : "",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "C_CITY", "C_REGION", "C_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "C_REGION", "C_NATION", "C_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ "D_YEAR" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 694224000000,
+ "partition_date_end" : 3153000000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.hbase.default.compression.codec" : "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
new file mode 100644
index 0000000..fc75ea8
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube2.json
@@ -0,0 +1,155 @@
+{
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851059",
+ "name" : "ssb_cube2",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "S_CITY",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_CITY",
+ "derived" : null
+ }, {
+ "name" : "S_REGION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_REGION",
+ "derived" : null
+ }, {
+ "name" : "S_NATION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_V_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "V_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : "",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST", "TOTAL_V_REVENUE" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ "D_YEAR" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 3153000000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.hbase.default.compression.codec" : "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
new file mode 100644
index 0000000..4509116
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube3.json
@@ -0,0 +1,143 @@
+{
+ "uuid" : "5c44df30-daec-486e-af90-927bf7851060",
+ "name" : "ssb_cube3",
+ "description" : "",
+ "dimensions" : [ {
+ "name" : "SSB.PART_DERIVED",
+ "table" : "SSB.PART",
+ "column" : null,
+ "derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+ }, {
+ "name" : "S_CITY",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_CITY",
+ "derived" : null
+ }, {
+ "name" : "S_REGION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_REGION",
+ "derived" : null
+ }, {
+ "name" : "S_NATION",
+ "table" : "SSB.SUPPLIER",
+ "column" : "S_NATION",
+ "derived" : null
+ }, {
+ "name" : "D_YEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_YEAR",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTH",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTH",
+ "derived" : null
+ }, {
+ "name" : "D_YEARMONTHNUM",
+ "table" : "SSB.DATES",
+ "column" : "D_YEARMONTHNUM",
+ "derived" : null
+ }, {
+ "name" : "D_WEEKNUMINYEAR",
+ "table" : "SSB.DATES",
+ "column" : "D_WEEKNUMINYEAR",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "_COUNT_",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_REVENUE",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_REVENUE",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TOTAL_SUPPLYCOST",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LO_SUPPLYCOST",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "LO_PARTKEY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_CITY",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_REGION",
+ "encoding" : "dict"
+ }, {
+ "column" : "S_NATION",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEAR",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTH",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_YEARMONTHNUM",
+ "encoding" : "dict"
+ }, {
+ "column" : "D_WEEKNUMINYEAR",
+ "encoding" : "dict"
+ } ]
+ },
+ "signature" : "",
+ "last_modified" : 1457503036686,
+ "model_name" : "ssb",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "_COUNT_", "TOTAL_REVENUE", "TOTAL_SUPPLYCOST"]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "LO_PARTKEY", "S_CITY", "S_REGION", "S_NATION", "D_YEAR", "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "S_REGION", "S_NATION", "S_CITY" ], [ "D_YEARMONTH", "D_YEARMONTHNUM", "D_WEEKNUMINYEAR" ] ],
+ "mandatory_dims" : [ "D_YEAR" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : [ ],
+ "status_need_notify" : [ ],
+ "partition_date_start" : 3153000000000,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.hbase.default.compression.codec" : "lz4",
+ "kylin.cube.aggrgroup.isMandatoryOnlyValid" : "true"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index 0e4a70c..f61492d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.controller;
import javax.servlet.http.HttpServletRequest;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
@@ -66,4 +67,11 @@ public class BasicController {
logger.error("", ex);
return new ErrorResponse(req.getRequestURL().toString(), ex);
}
+
+ protected void checkRequiredArg(String fieldName, Object fieldValue) {
+ if (fieldValue == null || StringUtils.isEmpty(String.valueOf(fieldValue))) {
+ throw new BadRequestException(fieldName + " is required");
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/server-base/src/main/java/org/apache/kylin/rest/controller/HybridController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/HybridController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/HybridController.java
new file mode 100644
index 0000000..18156b6
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/HybridController.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.util.Collection;
+
+import org.apache.kylin.rest.request.HybridRequest;
+import org.apache.kylin.rest.service.HybridService;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+@RequestMapping(value = "/hybrids")
+public class HybridController extends BasicController {
+
+ @Autowired
+ private HybridService hybridService;
+
+ @RequestMapping(value = "", method = RequestMethod.POST)
+ @ResponseBody
+ public HybridInstance create(@RequestBody HybridRequest request) {
+ checkRequiredArg("hybrid", request.getHybrid());
+ checkRequiredArg("project", request.getProject());
+ checkRequiredArg("model", request.getModel());
+ checkRequiredArg("cubes", request.getCubes());
+ HybridInstance instance = hybridService.createHybridCube(request.getHybrid(), request.getProject(), request.getModel(), request.getCubes());
+ return instance;
+ }
+
+ @RequestMapping(value = "", method = RequestMethod.PUT)
+ @ResponseBody
+ public HybridInstance update(@RequestBody HybridRequest request) {
+ checkRequiredArg("hybrid", request.getHybrid());
+ checkRequiredArg("project", request.getProject());
+ checkRequiredArg("model", request.getModel());
+ checkRequiredArg("cubes", request.getCubes());
+ HybridInstance instance = hybridService.updateHybridCube(request.getHybrid(), request.getProject(), request.getModel(), request.getCubes());
+ return instance;
+ }
+
+ @RequestMapping(value = "", method = RequestMethod.DELETE)
+ @ResponseBody
+ public void delete(@RequestBody HybridRequest request) {
+ checkRequiredArg("hybrid", request.getHybrid());
+ checkRequiredArg("project", request.getProject());
+ checkRequiredArg("model", request.getModel());
+ hybridService.deleteHybridCube(request.getHybrid(), request.getProject(), request.getModel());
+ }
+
+ @RequestMapping(value = "", method = RequestMethod.GET)
+ @ResponseBody
+ public Collection<HybridInstance> list(@RequestParam(required = false) String project, @RequestParam(required = false) String model) {
+ return hybridService.listHybrids(project, model);
+ }
+
+ @RequestMapping(value = "{hybrid}", method = RequestMethod.GET)
+ @ResponseBody
+ public HybridInstance get(@PathVariable String hybrid) {
+ return hybridService.getHybridInstance(hybrid);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/server-base/src/main/java/org/apache/kylin/rest/request/HybridRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/HybridRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/HybridRequest.java
new file mode 100644
index 0000000..68fbaa9
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/HybridRequest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.request;
+
+public class HybridRequest {
+
+ private String hybrid;
+
+ private String project;
+
+ private String model;
+
+ private String[] cubes;
+
+ public String getProject() {
+ return this.project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ public String getHybrid() {
+ return this.hybrid;
+ }
+
+ public void setHybrid(String hybrid) {
+ this.hybrid = hybrid;
+ }
+
+ public String getModel() {
+ return this.model;
+ }
+
+ public void setModel(String model) {
+ this.model = model;
+ }
+
+ public String[] getCubes() {
+ return this.cubes;
+ }
+
+ public void setCubes(String[] cubes) {
+ this.cubes = cubes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
new file mode 100644
index 0000000..dd1636a
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.tool.HybridCubeCLI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.access.prepost.PostFilter;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+@Component("hybridService")
+public class HybridService extends BasicService {
+
+ private static final Logger logger = LoggerFactory.getLogger(HybridService.class);
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or " + Constant.ACCESS_HAS_ROLE_MODELER)
+ public HybridInstance createHybridCube(String hybridName, String projectName, String modelName, String[] cubeNames) {
+ List<String> args = new ArrayList<String>();
+ args.add("-name");
+ args.add(hybridName);
+ args.add("-project");
+ args.add(projectName);
+ args.add("-model");
+ args.add(modelName);
+ args.add("-cubes");
+ args.add(StringUtils.join(cubeNames, ","));
+ args.add("-action");
+ args.add("create");
+ try {
+ HybridCubeCLI.main(args.toArray(new String[args.size()]));
+ } catch (Exception e) {
+ logger.warn("Create Hybrid Failed", e);
+ throw e;
+ }
+ return getHybridInstance(hybridName);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public HybridInstance updateHybridCube(String hybridName, String projectName, String modelName, String[] cubeNames) {
+ List<String> args = new ArrayList<String>();
+ args.add("-name");
+ args.add(hybridName);
+ args.add("-project");
+ args.add(projectName);
+ args.add("-model");
+ args.add(modelName);
+ args.add("-cubes");
+ args.add(StringUtils.join(cubeNames, ","));
+ args.add("-action");
+ args.add("update");
+ try {
+ HybridCubeCLI.main(args.toArray(new String[args.size()]));
+ } catch (Exception e) {
+ logger.warn("Update Hybrid Failed", e);
+ throw e;
+ }
+ return getHybridInstance(hybridName);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public void deleteHybridCube(String hybridName, String projectName, String modelName) {
+ List<String> args = new ArrayList<String>();
+ args.add("-name");
+ args.add(hybridName);
+ args.add("-project");
+ args.add(projectName);
+ args.add("-model");
+ args.add(modelName);
+ args.add("-action");
+ args.add("delete");
+ try {
+ HybridCubeCLI.main(args.toArray(new String[args.size()]));
+ } catch (Exception e) {
+ logger.warn("Delete Hybrid Failed", e);
+ throw e;
+ }
+ }
+
+ public HybridInstance getHybridInstance(String hybridName) {
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance(hybridName);
+ return hybridInstance;
+ }
+
+ @PostFilter(Constant.ACCESS_POST_FILTER_READ)
+ public List<HybridInstance> listHybrids(final String projectName, final String modelName) {
+ ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
+ List<HybridInstance> hybridsInProject = new ArrayList<HybridInstance>();
+
+ if (StringUtils.isEmpty(projectName)) {
+ hybridsInProject = new ArrayList(getHybridManager().listHybridInstances());
+ } else if (project == null) {
+ return new ArrayList<>();
+ } else {
+ List<RealizationEntry> realizationEntries = project.getRealizationEntries(RealizationType.HYBRID);
+ if (realizationEntries != null) {
+ for (RealizationEntry entry : realizationEntries) {
+ HybridInstance instance = getHybridManager().getHybridInstance(entry.getRealization());
+ hybridsInProject.add(instance);
+ }
+ }
+ }
+
+ DataModelDesc model = (null != modelName) ? getMetadataManager().getDataModelDesc(modelName) : null;
+ if (StringUtils.isEmpty(modelName)) {
+ return hybridsInProject;
+ } else if (model == null) {
+ return new ArrayList<>();
+ } else {
+ List<HybridInstance> hybridsInModel = new ArrayList<HybridInstance>();
+ for (HybridInstance hybridInstance : hybridsInProject) {
+ boolean hybridInModel = false;
+ for (RealizationEntry entry : hybridInstance.getRealizationEntries()) {
+ CubeDesc cubeDesc = getCubeDescManager().getCubeDesc(entry.getRealization());
+ if (cubeDesc != null && model.getName().equalsIgnoreCase(cubeDesc.getModel().getName())) {
+ hybridInModel = true;
+ break;
+ }
+ }
+ if (hybridInModel) {
+ hybridsInModel.add(hybridInstance);
+ }
+ }
+ return hybridsInModel;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 2f8667c..7dd7b0c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -84,7 +84,7 @@ public class ModelService extends BasicService {
public DataModelDesc createModelDesc(String projectName, DataModelDesc desc) throws IOException {
if (getMetadataManager().getDataModelDesc(desc.getName()) != null) {
- throw new InternalErrorException("The model named " + desc.getName() + " already exists");
+ throw new InternalErrorException("Model name " + desc.getName() + "is duplicated, could not create");
}
DataModelDesc createdDesc = null;
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
@@ -110,7 +110,7 @@ public class ModelService extends BasicService {
List<CubeDesc> cubeDescs = getCubeDescManager().listAllDesc();
for (CubeDesc cubeDesc : cubeDescs) {
if (cubeDesc.getModelName().equals(desc.getName())) {
- throw new InternalErrorException("Model referenced by cube,drop cubes under model and try again.");
+ throw new InternalErrorException("Model is referenced by Cube: " + cubeDesc.getName() + " , could not drop");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..f4afdf5 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -81,6 +81,8 @@
<shadedClassifierName>assembly</shadedClassifierName>
<artifactSet>
<includes>
+ <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
+ <include>org.apache.httpcomponents:httpcore</include>
<include>org.apache.kylin:kylin-tool</include>
</includes>
</artifactSet>
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java b/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java
new file mode 100644
index 0000000..8320213
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/HybridCubeCLI.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 1. Create new HybridCube
+ * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action create -name hybrid_name -project project_name -model model_name -cubes cube1,cube2
+ * 2. Update existing HybridCube
+ * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action update -name hybrid_name -project project_name -model model_name -cubes cube1,cube2,cube3
+ * 3. Delete the HybridCube
+ * bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action delete -name hybrid_name -project project_name -model model_name
+ */
+public class HybridCubeCLI extends AbstractApplication {
+
+ private static final Logger logger = LoggerFactory.getLogger(HybridCubeCLI.class);
+
+ private static final Option OPTION_ACTION = OptionBuilder.withArgName("action").hasArg().isRequired(true).withDescription("create/update/delete").create("action");
+
+ private static final Option OPTION_HYBRID_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("HybridCube name").create("name");
+
+ private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(true).withDescription("the target project for the hybrid cube").create("project");
+
+ private static final Option OPTION_MODEL = OptionBuilder.withArgName("model").hasArg().isRequired(true).withDescription("the target model for the hybrid cube").create("model");
+
+ private static final Option OPTION_CUBES = OptionBuilder.withArgName("cubes").hasArg().isRequired(false).withDescription("the cubes used in HybridCube, seperated by comma, empty if to delete HybridCube").create("cubes");
+
+ private final Options options;
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+ private HybridManager hybridManager;
+ private MetadataManager metadataManager;
+ private ResourceStore store;
+
+ public HybridCubeCLI() {
+ options = new Options();
+ options.addOption(OPTION_ACTION);
+ options.addOption(OPTION_HYBRID_NAME);
+ options.addOption(OPTION_PROJECT);
+ options.addOption(OPTION_MODEL);
+ options.addOption(OPTION_CUBES);
+
+ this.kylinConfig = KylinConfig.getInstanceFromEnv();
+ this.store = ResourceStore.getStore(kylinConfig);
+ this.cubeManager = CubeManager.getInstance(kylinConfig);
+ this.hybridManager = HybridManager.getInstance(kylinConfig);
+ this.metadataManager = MetadataManager.getInstance(kylinConfig);
+ }
+
+ public static void main(String[] args) {
+ HybridCubeCLI cli = new HybridCubeCLI();
+ cli.execute(args);
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ String action = optionsHelper.getOptionValue(OPTION_ACTION);
+ String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID_NAME);
+ String projectName = optionsHelper.getOptionValue(OPTION_PROJECT);
+ String modelName = optionsHelper.getOptionValue(OPTION_MODEL);
+ String cubeNamesStr = optionsHelper.getOptionValue(OPTION_CUBES);
+ String[] cubeNames = new String[] {};
+ if (cubeNamesStr != null)
+ cubeNames = cubeNamesStr.split(",");
+ String owner = null;
+
+ DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName);
+ if (modelDesc == null) {
+ throw new RuntimeException("Could not find model: " + modelName);
+ }
+
+ List<RealizationEntry> realizationEntries = new ArrayList<RealizationEntry>();
+ for (String cubeName : cubeNames) {
+ if (StringUtils.isEmpty(cubeName))
+ continue;
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ if (cube == null) {
+ throw new RuntimeException("Could not find cube: " + cubeName);
+ }
+ if (owner == null) {
+ owner = cube.getOwner();
+ }
+ realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cube.getName()));
+ }
+
+ HybridInstance hybridInstance = hybridManager.getHybridInstance(hybridName);
+ if ("create".equals(action)) {
+ if (hybridInstance != null) {
+ throw new RuntimeException("The Hybrid Cube does exist, could not create: " + hybridName);
+ }
+ //Create new Hybrid
+ create(hybridName, realizationEntries, projectName, owner);
+ } else if ("update".equals(action)) {
+ if (hybridInstance == null) {
+ throw new RuntimeException("The Hybrid Cube doesn't exist, could not update: " + hybridName);
+ }
+ // Update the Hybrid
+ update(hybridInstance, realizationEntries, projectName, owner);
+ } else if ("delete".equals(action)) {
+ if (hybridInstance == null) {
+ throw new RuntimeException("The Hybrid Cube doesn't exist, could not delete: " + hybridName);
+ }
+ // Delete the Hybrid
+ delete(hybridInstance);
+ }
+
+ }
+
+ private HybridInstance create(String hybridName, List<RealizationEntry> realizationEntries, String projectName, String owner) throws IOException {
+ checkSegmentOffset(realizationEntries);
+ HybridInstance hybridInstance = HybridInstance.create(kylinConfig, hybridName, realizationEntries);
+ store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
+ ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
+ hybridManager.reloadHybridInstance(hybridName);
+ logger.info("HybridInstance was created at: " + hybridInstance.getResourcePath());
+ return hybridInstance;
+ }
+
+ private void update(HybridInstance hybridInstance, List<RealizationEntry> realizationEntries, String projectName, String owner) throws IOException {
+ checkSegmentOffset(realizationEntries);
+ hybridInstance.setRealizationEntries(realizationEntries);
+ store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
+ ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
+ hybridManager.reloadHybridInstance(hybridInstance.getName());
+ logger.info("HybridInstance was updated at: " + hybridInstance.getResourcePath());
+ }
+
+ private void delete(HybridInstance hybridInstance) throws IOException {
+ ProjectManager.getInstance(kylinConfig).removeRealizationsFromProjects(RealizationType.HYBRID, hybridInstance.getName());
+ store.deleteResource(hybridInstance.getResourcePath());
+ hybridManager.reloadAllHybridInstance();
+ logger.info("HybridInstance was deleted at: " + hybridInstance.getResourcePath());
+ }
+
+ private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
+ if (realizationEntries == null || realizationEntries.size() == 0)
+ throw new RuntimeException("No realization found");
+ if (realizationEntries.size() == 1)
+ throw new RuntimeException("Hybrid needs at least 2 cubes");
+ long lastOffset = -1;
+ for (RealizationEntry entry : realizationEntries) {
+ if (entry.getType() != RealizationType.CUBE) {
+ throw new RuntimeException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
+ }
+
+ CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
+ CubeSegment segment = cubeInstance.getLastSegment();
+ if (segment == null)
+ continue;
+ if (lastOffset == -1) {
+ lastOffset = segment.getSourceOffsetEnd();
+ } else {
+ if (lastOffset > segment.getSourceOffsetStart()) {
+ throw new RuntimeException("Segments has overlap, could not hybrid. Last Segment End: " + lastOffset + ", Next Segment Start: " + segment.getSourceOffsetStart());
+ }
+ lastOffset = segment.getSourceOffsetEnd();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index 9dd4c81..1050bbe 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -26,15 +26,16 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -94,24 +95,24 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
}
private String getHttpResponse(String url) {
- HttpClient client = new HttpClient();
- String response = null;
+ DefaultHttpClient client = new DefaultHttpClient();
+ String msg = null;
int retry_times = 0;
- while (response == null && retry_times < HTTP_RETRY) {
+ while (msg == null && retry_times < HTTP_RETRY) {
retry_times++;
- HttpMethod get = new GetMethod(url);
+ HttpGet request = new HttpGet(url);
try {
- get.addRequestHeader("accept", "application/json");
- client.executeMethod(get);
- response = get.getResponseBodyAsString();
+ request.addHeader("accept", "application/json");
+ HttpResponse response = client.execute(request);
+ msg = EntityUtils.toString(response.getEntity());
} catch (Exception e) {
logger.warn("Failed to fetch http response. Retry={}", retry_times, e);
} finally {
- get.releaseConnection();
+ request.releaseConnection();
}
}
- return response;
+ return msg;
}
private void extractTaskCounter(String taskId, File exportDir, String taskUrl) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f6515d9/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
----------------------------------------------------------------------
diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
new file mode 100644
index 0000000..f4ed595
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class HybridCubeCLITest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void test1Create() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid");
+ Assert.assertNotNull(hybridInstance);
+ Assert.assertEquals("ssb_hybrid", hybridInstance.getName());
+ Assert.assertEquals(2, hybridInstance.getRealizationEntries().size());
+ }
+
+ @Test
+ public void test2Update() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid");
+ Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ Assert.assertEquals("ssb_hybrid", hybridInstance.getName());
+ Assert.assertEquals(2, hybridInstance.getRealizationEntries().size());
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2,ssb_cube3", "-action", "update" });
+
+ hybridInstance = hybridManager.getHybridInstance("ssb_hybrid");
+ Assert.assertNotNull(hybridInstance);
+ Assert.assertEquals("ssb_hybrid", hybridInstance.getName());
+ Assert.assertEquals(3, hybridInstance.getRealizationEntries().size());
+ }
+
+ @Test
+ public void test3Delete() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+ Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-action", "delete" });
+
+ HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid");
+ Assert.assertNull(hybridInstance);
+ Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size());
+ }
+
+}