You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/17 09:38:59 UTC
[2/2] kylin git commit: KYLIN_2752 code clean up
KYLIN_2752 code clean up
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/876811f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/876811f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/876811f2
Branch: refs/heads/master
Commit: 876811f2312981b99344cd700e1300a0462ec124
Parents: c9a855f
Author: Li Yang <li...@apache.org>
Authored: Sun Sep 17 16:21:12 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Sep 17 16:28:19 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManagerTest.java | 12 ++++++++----
.../metadata/realization/IRealizationConstants.java | 4 ----
examples/test_case_data/localmeta/kylin.properties | 6 ------
.../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java | 7 ++++---
.../org/apache/kylin/storage/hbase/util/HBaseClean.java | 5 +++--
.../org/apache/kylin/storage/hbase/util/HBaseUsage.java | 5 +++--
.../kylin/storage/hbase/util/OrphanHBaseCleanJob.java | 5 +++--
.../kylin/storage/hbase/util/StorageCleanupJob.java | 9 +++++----
8 files changed, 26 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index da27032..0188904 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -307,10 +307,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
@Test
public void testGetCubeNameWithNamespace() {
- CubeManager mgr = CubeManager.getInstance(getTestConfig());
- String tablename = mgr.generateStorageLocation();
- System.out.println(tablename);
- assertTrue(tablename.startsWith("HELLO:WORLD"));
+ System.setProperty("kylin.storage.hbase.table-name-prefix", "HELLO_");
+ try {
+ CubeManager mgr = CubeManager.getInstance(getTestConfig());
+ String tablename = mgr.generateStorageLocation();
+ assertTrue(tablename.startsWith("HELLO_"));
+ } finally {
+ System.clearProperty("kylin.storage.hbase.table-name-prefix");
+ }
}
public CubeDescManager getCubeDescManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 2a3b01e..2e4f400 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -22,10 +22,6 @@ package org.apache.kylin.metadata.realization;
*/
public class IRealizationConstants {
- public final static String SharedHbaseStorageLocationPrefix = "KYLIN_";
- public final static String CubeHbaseStorageLocationPrefix = "KYLIN_";
- public final static String IIHbaseStorageLocationPrefix = "KYLIN_II_";
-
/**
* For each cube htable, we leverage htable's metadata to keep track of
* which kylin server(represented by its kylin_metadata prefix) owns this htable
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 48e2856..c7dda3f 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,12 +39,6 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# The prefix for cube table
-kylin.storage.hbase.tableprefix=WORLD_
-
-# The namespace for hbase storage
-kylin.storage.hbase.namespace=HELLO
-
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 3728ea1..aceebb4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -46,15 +46,16 @@ public class StorageCleanJobHbaseUtil {
public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {
Configuration conf = HBaseConfiguration.create();
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
+ if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
//only take care htables that belongs to self, and created more than 2 days
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..1bafeb2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -57,9 +57,10 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..bb59242 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -43,9 +43,10 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..0c35e46 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -56,10 +56,11 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
// get all kylin hbase tables
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..2ae21b4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -78,16 +78,17 @@ public class StorageCleanupJob extends AbstractApplication {
protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
+ if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
//only take care htables that belongs to self, and created more than 2 days
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}