You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/10/08 02:26:07 UTC
[45/50] [abbrv] kylin git commit: KYLIN-2059 remove the time
threshold on storage cleanup
KYLIN-2059 remove the time threshold on storage cleanup
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95ebc851
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95ebc851
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95ebc851
Branch: refs/heads/orderedbytes
Commit: 95ebc851274953023ba092a249c1ddbbee5cd845
Parents: 334c2e0
Author: shaofengshi <sh...@apache.org>
Authored: Fri Sep 30 09:13:03 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 6 14:44:05 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 ----
.../kylin/engine/mr/steps/MetadataCleanupJob.java | 2 +-
.../kylin/storage/hbase/util/StorageCleanupJob.java | 15 ++++-----------
3 files changed, 5 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4d1639b..99c3c5a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -780,10 +780,6 @@ abstract public class KylinConfigBase implements Serializable {
return Long.parseLong(getOptional("kylin.query.hbase.hconnection.threads.alive.seconds", "60"));
}
- public long getStorageCleanupTimeThreshold() {
- return Long.valueOf(this.getOptional("kylin.storage.cleanup.time.threshold", "172800000")); //default two days
- }
-
public int getAppendDictEntrySize() {
return Integer.parseInt(getOptional("kylin.dict.append.entry.size", "10000000"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index 962697e..f2b1d6b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -54,7 +54,7 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
private KylinConfig config = null;
- public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000L; // 2 days
+ public static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 * 1000L; // 30 days
/*
http://git-wip-us.apache.org/repos/asf/kylin/blob/95ebc851/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index dffce36..2c2f11c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -34,7 +34,6 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -63,10 +62,10 @@ public class StorageCleanupJob extends AbstractApplication {
@SuppressWarnings("static-access")
protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete");
- protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning: will delete any intermediate hive tables").create("force");
+ protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning: will delete all kylin intermediate hive tables").create("force");
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
- public static final int TIME_THRESHOLD_DELETE_HTABLE = 10; // Unit minute
+ public static final int deleteTimeout = 10; // Unit minute
protected boolean delete = false;
protected boolean force = false;
@@ -74,7 +73,6 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold();
// get all kylin hbase tables
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
@@ -82,14 +80,9 @@ public class StorageCleanupJob extends AbstractApplication {
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
- String creationTime = desc.getValue(IRealizationConstants.HTableCreationTime);
if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
//only take care htables that belongs to self, and created more than 2 days
- if (StringUtils.isEmpty(creationTime) || (System.currentTimeMillis() - Long.valueOf(creationTime) > TIME_THREADSHOLD)) {
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
- } else {
- logger.info("Exclude table " + desc.getTableName().getNameAsString() + " from drop list, as it is newly created");
- }
}
}
@@ -111,9 +104,9 @@ public class StorageCleanupJob extends AbstractApplication {
FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin, htableName));
executorService.execute(futureTask);
try {
- futureTask.get(TIME_THRESHOLD_DELETE_HTABLE, TimeUnit.MINUTES);
+ futureTask.get(deleteTimeout, TimeUnit.MINUTES);
} catch (TimeoutException e) {
- logger.warn("It fails to delete htable " + htableName + ", for it cost more than " + TIME_THRESHOLD_DELETE_HTABLE + " minutes!");
+ logger.warn("It fails to delete htable " + htableName + ", for it cost more than " + deleteTimeout + " minutes!");
futureTask.cancel(true);
} catch (Exception e) {
e.printStackTrace();