You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/17 08:29:40 UTC
[3/4] kylin git commit: KYLIN_2752 Make cube hbase tablename prefix
configurable
KYLIN_2752 Make cube hbase tablename prefix configurable
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9a855f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9a855f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9a855f2
Branch: refs/heads/KYLIN-2846
Commit: c9a855f271d592262b6e9ede0262ae160fcc42cd
Parents: 87ccbc3
Author: Albert Lee <al...@gmail.com>
Authored: Sun Sep 10 23:23:13 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Sep 17 16:21:33 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
core-common/src/main/resources/kylin-defaults.properties | 3 +++
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 8 ++++----
.../src/test/java/org/apache/kylin/cube/CubeManagerTest.java | 8 ++++++++
examples/test_case_data/localmeta/kylin.properties | 6 ++++++
5 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c74c093..71dbfbe 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -720,6 +720,10 @@ abstract public class KylinConfigBase implements Serializable {
return StorageURL.valueOf(url);
}
+ public String getHBaseTableNamePrefix() {
+ return getOptional("kylin.storage.hbase.table-name-prefix", "KYLIN_");
+ }
+
public String getHBaseClusterFs() {
return getOptional("kylin.storage.hbase.cluster-fs", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index c422b42..141fb7b 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -74,6 +74,9 @@ kylin.source.hive.redistribute-flat-table=true
# The storage for final cube file in hbase
kylin.storage.url=hbase
+# The prefix of hbase table
+kylin.storage.hbase.table-name-prefix=KYLIN_
+
# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
kylin.storage.hbase.compression-codec=none
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 5fd3d1b..425e32f 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -65,7 +65,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.IRealizationProvider;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -75,6 +74,7 @@ import org.apache.kylin.source.SourcePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -683,8 +683,9 @@ public class CubeManager implements IRealizationProvider {
return segment;
}
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
+ @VisibleForTesting
+ /*private*/ String generateStorageLocation() {
+ String namePrefix = config.getHBaseTableNamePrefix();
String tableName = "";
Random ran = new Random();
do {
@@ -695,7 +696,6 @@ public class CubeManager implements IRealizationProvider {
}
tableName = sb.toString();
} while (this.usedStorageLocation.containsValue(tableName));
-
return tableName;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 7dd7212..da27032 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -305,6 +305,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
assertTrue((Long) mergedSeg.start.v == 0 && (Long) mergedSeg.end.v == 8000);
}
+ @Test
+ public void testGetCubeNameWithNamespace() {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ String tablename = mgr.generateStorageLocation();
+ System.out.println(tablename);
+ assertTrue(tablename.startsWith("HELLO:WORLD"));
+ }
+
public CubeDescManager getCubeDescManager() {
return CubeDescManager.getInstance(getTestConfig());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index c7dda3f..48e2856 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,6 +39,12 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase
+# The prefix for cube table
+kylin.storage.hbase.tableprefix=WORLD_
+
+# The namespace for hbase storage
+kylin.storage.hbase.namespace=HELLO
+
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin