You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/17 09:38:58 UTC
[1/2] kylin git commit: KYLIN_2752 Make cube hbase tablename prefix
configurable
Repository: kylin
Updated Branches:
refs/heads/master 87ccbc3a6 -> 876811f23
KYLIN_2752 Make cube hbase tablename prefix configurable
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9a855f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9a855f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9a855f2
Branch: refs/heads/master
Commit: c9a855f271d592262b6e9ede0262ae160fcc42cd
Parents: 87ccbc3
Author: Albert Lee <al...@gmail.com>
Authored: Sun Sep 10 23:23:13 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Sep 17 16:21:33 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
core-common/src/main/resources/kylin-defaults.properties | 3 +++
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 8 ++++----
.../src/test/java/org/apache/kylin/cube/CubeManagerTest.java | 8 ++++++++
examples/test_case_data/localmeta/kylin.properties | 6 ++++++
5 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c74c093..71dbfbe 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -720,6 +720,10 @@ abstract public class KylinConfigBase implements Serializable {
return StorageURL.valueOf(url);
}
+ public String getHBaseTableNamePrefix() {
+ return getOptional("kylin.storage.hbase.table-name-prefix", "KYLIN_");
+ }
+
public String getHBaseClusterFs() {
return getOptional("kylin.storage.hbase.cluster-fs", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index c422b42..141fb7b 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -74,6 +74,9 @@ kylin.source.hive.redistribute-flat-table=true
# The storage for final cube file in hbase
kylin.storage.url=hbase
+# The prefix of hbase table
+kylin.storage.hbase.table-name-prefix=KYLIN_
+
# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
kylin.storage.hbase.compression-codec=none
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 5fd3d1b..425e32f 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -65,7 +65,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.IRealizationProvider;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -75,6 +74,7 @@ import org.apache.kylin.source.SourcePartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -683,8 +683,9 @@ public class CubeManager implements IRealizationProvider {
return segment;
}
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
+ @VisibleForTesting
+ /*private*/ String generateStorageLocation() {
+ String namePrefix = config.getHBaseTableNamePrefix();
String tableName = "";
Random ran = new Random();
do {
@@ -695,7 +696,6 @@ public class CubeManager implements IRealizationProvider {
}
tableName = sb.toString();
} while (this.usedStorageLocation.containsValue(tableName));
-
return tableName;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 7dd7212..da27032 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -305,6 +305,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
assertTrue((Long) mergedSeg.start.v == 0 && (Long) mergedSeg.end.v == 8000);
}
+ @Test
+ public void testGetCubeNameWithNamespace() {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ String tablename = mgr.generateStorageLocation();
+ System.out.println(tablename);
+ assertTrue(tablename.startsWith("HELLO:WORLD"));
+ }
+
public CubeDescManager getCubeDescManager() {
return CubeDescManager.getInstance(getTestConfig());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9a855f2/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index c7dda3f..48e2856 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,6 +39,12 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase
+# The prefix for cube table
+kylin.storage.hbase.tableprefix=WORLD_
+
+# The namespace for hbase storage
+kylin.storage.hbase.namespace=HELLO
+
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
[2/2] kylin git commit: KYLIN_2752 code clean up
Posted by li...@apache.org.
KYLIN_2752 code clean up
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/876811f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/876811f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/876811f2
Branch: refs/heads/master
Commit: 876811f2312981b99344cd700e1300a0462ec124
Parents: c9a855f
Author: Li Yang <li...@apache.org>
Authored: Sun Sep 17 16:21:12 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Sep 17 16:28:19 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManagerTest.java | 12 ++++++++----
.../metadata/realization/IRealizationConstants.java | 4 ----
examples/test_case_data/localmeta/kylin.properties | 6 ------
.../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java | 7 ++++---
.../org/apache/kylin/storage/hbase/util/HBaseClean.java | 5 +++--
.../org/apache/kylin/storage/hbase/util/HBaseUsage.java | 5 +++--
.../kylin/storage/hbase/util/OrphanHBaseCleanJob.java | 5 +++--
.../kylin/storage/hbase/util/StorageCleanupJob.java | 9 +++++----
8 files changed, 26 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index da27032..0188904 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -307,10 +307,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
@Test
public void testGetCubeNameWithNamespace() {
- CubeManager mgr = CubeManager.getInstance(getTestConfig());
- String tablename = mgr.generateStorageLocation();
- System.out.println(tablename);
- assertTrue(tablename.startsWith("HELLO:WORLD"));
+ System.setProperty("kylin.storage.hbase.table-name-prefix", "HELLO_");
+ try {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ String tablename = mgr.generateStorageLocation();
+ assertTrue(tablename.startsWith("HELLO_"));
+ } finally {
+ System.clearProperty("kylin.storage.hbase.table-name-prefix");
+ }
}
public CubeDescManager getCubeDescManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 2a3b01e..2e4f400 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -22,10 +22,6 @@ package org.apache.kylin.metadata.realization;
*/
public class IRealizationConstants {
- public final static String SharedHbaseStorageLocationPrefix = "KYLIN_";
- public final static String CubeHbaseStorageLocationPrefix = "KYLIN_";
- public final static String IIHbaseStorageLocationPrefix = "KYLIN_II_";
-
/**
* For each cube htable, we leverage htable's metadata to keep track of
* which kylin server(represented by its kylin_metadata prefix) owns this htable
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 48e2856..c7dda3f 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,12 +39,6 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# The prefix for cube table
-kylin.storage.hbase.tableprefix=WORLD_
-
-# The namespace for hbase storage
-kylin.storage.hbase.namespace=HELLO
-
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 3728ea1..aceebb4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -46,15 +46,16 @@ public class StorageCleanJobHbaseUtil {
public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {
Configuration conf = HBaseConfiguration.create();
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
+ if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
//only take care htables that belongs to self, and created more than 2 days
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..1bafeb2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -57,9 +57,10 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..bb59242 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -43,9 +43,10 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..0c35e46 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -56,10 +56,11 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
// get all kylin hbase tables
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..2ae21b4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -78,16 +78,17 @@ public class StorageCleanupJob extends AbstractApplication {
protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
+ if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
//only take care htables that belongs to self, and created more than 2 days
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}