You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/08/11 03:04:33 UTC
[kylin] branch master updated: KYLIN-4686 clean metadata support to
delete all jobs
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 2e937c6 KYLIN-4686 clean metadata support to delete all jobs
2e937c6 is described below
commit 2e937c62990ecbc9f77f6f377b7bd4cf1dbdc230
Author: chuxiao <ch...@didichuxing.com>
AuthorDate: Thu Aug 6 11:10:20 2020 +0800
KYLIN-4686 clean metadata support to delete all jobs
---
.../main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java | 4 ++--
.../java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java | 4 ++--
tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java | 9 ++++++++-
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
index 78302bb..a9a7f12 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -80,7 +80,7 @@ public class MetadataCleanupJob {
}
// function entrance
- public Map<String, Long> cleanup(boolean delete, int jobOutdatedDays) throws Exception {
+ public Map<String, Long> cleanup(boolean delete, int jobOutdatedDays, boolean deleteAllJobs) throws Exception {
CubeManager cubeManager = CubeManager.getInstance(config);
long newResourceTimeCut = System.currentTimeMillis() - NEW_RESOURCE_THREADSHOLD_MS;
FileSystem fs = HadoopUtil.getWorkingFileSystem(HadoopUtil.getCurrentConfiguration());
@@ -168,7 +168,7 @@ public class MetadataCleanupJob {
List<ExecutablePO> allExecutable = executableDao.getJobs();
for (ExecutablePO executable : allExecutable) {
long lastModified = executable.getLastModified();
- if (lastModified < outdatedJobTimeCut && isJobComplete(executableDao, executable)) {
+ if (lastModified < outdatedJobTimeCut && (deleteAllJobs || isJobComplete(executableDao, executable))) {
String jobResPath = ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid();
String jobOutputResPath = ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid();
long outputLastModified = getTimestamp(jobOutputResPath);
diff --git a/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
index ad819e6..bed1a9c 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
@@ -45,7 +45,7 @@ public class MetadataCleanupJobTest {
// file resource store may lose timestamp precision with millis second, set last modified as 2000
staticCreateTestMetadata(false, new ResetTimeHook(2000, "src/test/resources/test_meta"));
MetadataCleanupJob metadataCleanupJob = new MetadataCleanupJob();
- Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30);
+ Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30, false);
Assert.assertEquals(7, cleanupMap.size());
for (long timestamp : cleanupMap.values()) {
Assert.assertEquals(2000, timestamp);
@@ -56,7 +56,7 @@ public class MetadataCleanupJobTest {
public void testNotCleanUp() throws Exception {
staticCreateTestMetadata(false, new ResetTimeHook(System.currentTimeMillis(), "src/test/resources/test_meta"));
MetadataCleanupJob metadataCleanupJob = new MetadataCleanupJob();
- Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30);
+ Map<String, Long> cleanupMap = metadataCleanupJob.cleanup(false, 30, false);
Assert.assertEquals(0, cleanupMap.size());
}
diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
index 1e5214d..1cb5df7 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
@@ -39,6 +39,11 @@ public class MetadataCleanupJob extends AbstractApplication {
.isRequired(false).withDescription("Specify how many days of job metadata keeping. Default 30 days")
.create("jobThreshold");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DELETE_ALL_JOBS = OptionBuilder.withArgName("deleteAllJobs").hasArg()
+ .isRequired(false).withDescription("Delte all jobs include error jobs. Default false")
+ .create("deleteAllJobs");
+
public static void main(String[] args) throws Exception {
new MetadataCleanupJob().execute(args);
}
@@ -48,6 +53,7 @@ public class MetadataCleanupJob extends AbstractApplication {
Options options = new Options();
options.addOption(OPTION_DELETE);
options.addOption(OPTION_THRESHOLD_FOR_JOB);
+ options.addOption(OPTION_DELETE_ALL_JOBS);
return options;
}
@@ -59,7 +65,8 @@ public class MetadataCleanupJob extends AbstractApplication {
int jobOutdatedDays = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB)
? Integer.parseInt(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB))
: DEFAULT_JOB_OUTDATED_THRESHOLD_DAYS;
+ boolean deleteAllJobs = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE_ALL_JOBS));
- new org.apache.kylin.rest.job.MetadataCleanupJob().cleanup(delete, jobOutdatedDays);
+ new org.apache.kylin.rest.job.MetadataCleanupJob().cleanup(delete, jobOutdatedDays, deleteAllJobs);
}
}