You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/10/23 09:17:11 UTC
[1/4] kylin git commit: KYLIN-2846 update unit test
Repository: kylin
Updated Branches:
refs/heads/master 48f3fb195 -> febbf2623
KYLIN-2846 update unit test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/febbf262
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/febbf262
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/febbf262
Branch: refs/heads/master
Commit: febbf26230dec27eebe0d6568e2820ab26596b57
Parents: dbf59e5
Author: shaofengshi <sh...@apache.org>
Authored: Sat Oct 21 23:25:41 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 23 17:16:27 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 4 +++-
.../test/java/org/apache/kylin/cube/CubeManagerTest.java | 10 ++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/febbf262/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index e24aa33..388c840 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -679,7 +679,9 @@ public class CubeManager implements IRealizationProvider {
Random ran = new Random();
do {
StringBuffer sb = new StringBuffer();
- sb.append(namespace).append(":");
+ if ((namespace.equals("default") || namespace.equals("")) == false) {
+ sb.append(namespace).append(":");
+ }
sb.append(namePrefix);
for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/febbf262/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index d8d48f7..3f68bab 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -314,8 +314,18 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
} finally {
System.clearProperty("kylin.storage.hbase.table-name-prefix");
}
+
+ System.setProperty("kylin.storage.hbase.namespace", "MYSPACE");
+ try {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ String tablename = mgr.generateStorageLocation();
+ assertTrue(tablename.startsWith("MYSPACE:"));
+ } finally {
+ System.clearProperty("kylin.storage.hbase.namespace");
+ }
}
+
public CubeDescManager getCubeDescManager() {
return CubeDescManager.getInstance(getTestConfig());
}
[4/4] kylin git commit: Add a config of hbase namespace for cube
storage
Posted by sh...@apache.org.
Add a config of hbase namespace for cube storage
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09cc2edc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09cc2edc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09cc2edc
Branch: refs/heads/master
Commit: 09cc2edc4bf56bff986aeb00d431ef79c59d03e2
Parents: 48f3fb1
Author: Liu Shaohui <li...@xiaomi.com>
Authored: Tue Sep 5 15:50:29 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 23 17:16:27 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
core-common/src/main/resources/kylin-defaults.properties | 3 +++
core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 2 ++
.../java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-
.../java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-
5 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/09cc2edc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d204f71..0faea9c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -780,6 +780,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.storage.hbase.table-name-prefix", "KYLIN_");
}
+ public String getHBaseStorageNameSpace() {
+ return getOptional("kylin.storage.hbase.namespace", "default");
+ }
+
public String getHBaseClusterFs() {
return getOptional("kylin.storage.hbase.cluster-fs", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/09cc2edc/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index e3632ae..1602087 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -77,6 +77,9 @@ kylin.storage.url=hbase
# The prefix of hbase table
kylin.storage.hbase.table-name-prefix=KYLIN_
+# The namespace for hbase storage
+kylin.storage.hbase.namespace=default
+
# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
kylin.storage.hbase.compression-codec=none
http://git-wip-us.apache.org/repos/asf/kylin/blob/09cc2edc/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b18d40f..e24aa33 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -674,10 +674,12 @@ public class CubeManager implements IRealizationProvider {
@VisibleForTesting
/*private*/ String generateStorageLocation() {
String namePrefix = config.getHBaseTableNamePrefix();
+ String namespace = config.getHBaseStorageNameSpace();
String tableName = "";
Random ran = new Random();
do {
StringBuffer sb = new StringBuffer();
+ sb.append(namespace).append(":");
sb.append(namePrefix);
for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/09cc2edc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 30616c5..d1caccb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -47,7 +47,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
options.addOption(OPTION_CUBE_NAME);
parseOptions(options, args);
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
+ String tableName = getOptionValue(OPTION_HTABLE_NAME);
// e.g
// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
// end with "/"
http://git-wip-us.apache.org/repos/asf/kylin/blob/09cc2edc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 7dc67e5..8de772f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -67,7 +67,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+ String cubeName = getOptionValue(OPTION_CUBE_NAME);
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
[2/4] kylin git commit: Addium for KYLIN-2846
Posted by sh...@apache.org.
Addium for KYLIN-2846
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3d3585e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3d3585e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3d3585e
Branch: refs/heads/master
Commit: f3d3585ed6360657574ac5b89535b18e41ac3efa
Parents: 09cc2ed
Author: Liu Shaohui <li...@xiaomi.com>
Authored: Fri Sep 15 14:34:18 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 23 17:16:27 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f3d3585e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 8de772f..5f51c13 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -91,7 +91,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
attachCubeMetadata(cube, job.getConfiguration());
Configuration hbaseConf = HBaseConfiguration.create(getConf());
- HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase());
+ HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));
// Automatic config !
HFileOutputFormat3.configureIncrementalLoad(job, htable);
[3/4] kylin git commit: KYLIN-2846 use configured hbase namespace in
StorageCleanJobHbaseUtil
Posted by sh...@apache.org.
KYLIN-2846 use configured hbase namespace in StorageCleanJobHbaseUtil
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbf59e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbf59e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbf59e5c
Branch: refs/heads/master
Commit: dbf59e5cd06ad7bb45f22d89478bba7f19618608
Parents: f3d3585
Author: zeng.yaowu <51...@qq.com>
Authored: Sat Oct 21 23:05:17 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 23 17:16:27 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbf59e5c/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index aceebb4..9933fb4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -50,7 +50,9 @@ public class StorageCleanJobHbaseUtil {
CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
- String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
+ String namespace = kylinConfig.getHBaseStorageNameSpace();
+ String tableNamePrefix = (namespace.equals("default") || namespace.equals(""))
+ ? kylinConfig.getHBaseTableNamePrefix() : (namespace + ":" + kylinConfig.getHBaseTableNamePrefix());
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {