You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/07 13:20:55 UTC
[20/28] incubator-kylin git commit: KYLIN-991 Add timestamp for
htables, and check the time before cleanup
KYLIN-991 Add timestamp for htables, and check the time before cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c7990735
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c7990735
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c7990735
Branch: refs/heads/2.x-staging
Commit: c7990735c6f901d64adbd8927afd57fd63a69e51
Parents: d2067c4
Author: shaofengshi <sh...@apache.org>
Authored: Sun Sep 6 17:16:51 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Sep 6 17:16:51 2015 +0800
----------------------------------------------------------------------
.../realization/IRealizationConstants.java | 3 +++
.../kylin/job/hadoop/cube/StorageCleanupJob.java | 19 ++++++++++++-------
.../hadoop/invertedindex/IICreateHTableJob.java | 1 +
.../storage/hbase/steps/CubeHTableUtil.java | 1 +
4 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c7990735/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 0f2c012..7cbcdf8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -32,4 +32,7 @@ public class IRealizationConstants {
*/
public final static String HTableTag = "KYLIN_HOST";
+
+ public final static String HTableCreationTime = "CREATION_TIME";
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c7990735/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
index 53b489e..489b9ed 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
@@ -18,14 +18,10 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -52,6 +48,11 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
/**
* @author ysong1
*/
@@ -62,6 +63,8 @@ public class StorageCleanupJob extends AbstractHadoopJob {
protected static final Logger log = LoggerFactory.getLogger(StorageCleanupJob.class);
+ public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000l; // 2 days
+
boolean delete = false;
protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
@@ -108,9 +111,11 @@ public class StorageCleanupJob extends AbstractHadoopJob {
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
+ String creationTime = desc.getValue(IRealizationConstants.HTableCreationTime);
if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
- //only take care htables that belongs to self
- allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+ //only take care htables that belongs to self, and created more than 2 days
+ if (StringUtils.isNotEmpty(creationTime) || (System.currentTimeMillis() - Long.valueOf(creationTime) > TIME_THREADSHOLD))
+ allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c7990735/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 258f519..22ed3a2 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -99,6 +99,7 @@ public class IICreateHTableJob extends AbstractHadoopJob {
cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
tableDesc.addFamily(cf);
tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());
+ tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
Configuration conf = HBaseConfiguration.create(getConf());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c7990735/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 7b0dad9..221be8a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -35,6 +35,7 @@ public class CubeHTableUtil {
// https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html
tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());
tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
+ tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
Configuration conf = HBaseConfiguration.create();
HBaseAdmin admin = new HBaseAdmin(conf);