You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by zi...@apache.org on 2022/04/28 00:50:01 UTC
[gobblin] branch master updated: [GOBBLIN-1636] Close DatasetCleaner after clean task (#3497)
This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 3658ec40c [GOBBLIN-1636] Close DatasetCleaner after clean task (#3497)
3658ec40c is described below
commit 3658ec40c3dc564de474aad408fbf7caa7d4b592
Author: Zihan Li <zi...@linkedin.com>
AuthorDate: Wed Apr 27 17:49:55 2022 -0700
[GOBBLIN-1636] Close DatasetCleaner after clean task (#3497)
* address comments
* use connectionmanager when httpclient is not cloesable
* [GOBBLIN-1636] Close DatasetCleaner after clean task
Co-authored-by: Zihan Li <zi...@zihli-mn2.linkedin.biz>
---
.../apache/gobblin/runtime/retention/DatasetCleanerCli.java | 11 ++++++++++-
.../gobblin/runtime/retention/DatasetCleanerTask.java | 13 +++++++++++--
.../data/management/retention/DatasetCleanerJob.java | 6 +++++-
3 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerCli.java b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerCli.java
index 47b4cf035..1bf360a42 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerCli.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerCli.java
@@ -43,12 +43,21 @@ public class DatasetCleanerCli implements CliApplication {
@Override
public void run(String[] args) {
+ DatasetCleaner datasetCleaner = null;
try {
Properties properties = readProperties(parseConfigLocation(args));
- DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(new Configuration()), properties);
+ datasetCleaner = new DatasetCleaner(FileSystem.get(new Configuration()), properties);
datasetCleaner.clean();
} catch (IOException e) {
throw new RuntimeException(e);
+ } finally {
+ if (datasetCleaner != null) {
+ try {
+ datasetCleaner.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
}
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
index 719ecc163..a3ab7654f 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
@@ -47,16 +47,25 @@ public class DatasetCleanerTask extends BaseAbstractTask {
}
@Override
- public void run() {
+ public void run(){
+ DatasetCleaner datasetCleaner = null;
try {
Configuration conf = new Configuration();
JobConfigurationUtils.putStateIntoConfiguration(this.taskContext.getTaskState(), conf);
- DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(conf), this.taskContext.getTaskState().getProperties());
+ datasetCleaner = new DatasetCleaner(FileSystem.get(conf), this.taskContext.getTaskState().getProperties());
datasetCleaner.clean();
this.workingState = WorkUnitState.WorkingState.SUCCESSFUL;
} catch (IOException e) {
this.workingState = WorkUnitState.WorkingState.FAILED;
throw new RuntimeException(e);
+ } finally {
+ if (datasetCleaner != null) {
+ try {
+ datasetCleaner.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
}
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleanerJob.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleanerJob.java
index 3a5cc3797..5833e3daa 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleanerJob.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleanerJob.java
@@ -61,7 +61,11 @@ public class DatasetCleanerJob extends AbstractJob implements Tool {
@Override
public void run() throws Exception {
if (this.datasetCleaner != null) {
- this.datasetCleaner.clean();
+ try {
+ this.datasetCleaner.clean();
+ } finally {
+ this.datasetCleaner.close();
+ }
}
}