You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/10 10:56:14 UTC
kylin git commit: KYLIN-2846 Add a config of hbase namespace for cube
storage
Repository: kylin
Updated Branches:
refs/heads/master d65376b58 -> ffcbdb7f4
KYLIN-2846 Add a config of hbase namespace for cube storage
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ffcbdb7f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ffcbdb7f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ffcbdb7f
Branch: refs/heads/master
Commit: ffcbdb7f4288fa6011f018163bdce9602415cf10
Parents: d65376b
Author: Liu Shaohui <li...@xiaomi.com>
Authored: Tue Sep 5 15:50:29 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Sep 10 18:55:54 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
core-common/src/main/resources/kylin-defaults.properties | 3 +++
core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 2 ++
.../java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-
.../java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-
5 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ffcbdb7f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 990736f..8f5d495 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -716,6 +716,10 @@ abstract public class KylinConfigBase implements Serializable {
return StorageURL.valueOf(url);
}
+ public String getHBaseStorageNameSpace() {
+ return getOptional("kylin.storage.hbase.namespace", "default");
+ }
+
public String getHBaseClusterFs() {
return getOptional("kylin.storage.hbase.cluster-fs", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ffcbdb7f/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index e818dc1..815d7d0 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -65,6 +65,9 @@ kylin.source.hive.redistribute-flat-table=true
# The storage for final cube file in hbase
kylin.storage.url=hbase
+# The namespace for hbase storage
+kylin.storage.hbase.namespace=default
+
# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
kylin.storage.hbase.compression-codec=none
http://git-wip-us.apache.org/repos/asf/kylin/blob/ffcbdb7f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 5fd3d1b..606ce4d 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -684,11 +684,13 @@ public class CubeManager implements IRealizationProvider {
}
private String generateStorageLocation() {
+ String namespace = config.getHBaseStorageNameSpace();
String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
String tableName = "";
Random ran = new Random();
do {
StringBuffer sb = new StringBuffer();
+ sb.append(namespace).append(":");
sb.append(namePrefix);
for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/ffcbdb7f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 30616c5..d1caccb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -47,7 +47,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
options.addOption(OPTION_CUBE_NAME);
parseOptions(options, args);
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
+ String tableName = getOptionValue(OPTION_HTABLE_NAME);
// e.g
// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
// end with "/"
http://git-wip-us.apache.org/repos/asf/kylin/blob/ffcbdb7f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 1a624c4..9f9b382 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -68,7 +68,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+ String cubeName = getOptionValue(OPTION_CUBE_NAME);
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());