You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/04/12 02:13:49 UTC
[incubator-inlong] branch master updated: [INLONG-3587][Agent] Auto close file resource after list (#3588)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5d4806bd8 [INLONG-3587][Agent] Auto close file resource after list (#3588)
5d4806bd8 is described below
commit 5d4806bd8f73bc98f06062be7934a403a7c1546f
Author: huzk <10...@qq.com>
AuthorDate: Tue Apr 12 10:13:45 2022 +0800
[INLONG-3587][Agent] Auto close file resource after list (#3588)
---
.../org/apache/inlong/agent/db/LocalProfile.java | 25 +++++++++++++---------
.../agent/plugin/trigger/DirectoryTrigger.java | 10 ++++++---
.../inlong/agent/plugin/trigger/PathPattern.java | 20 +++++++++++------
3 files changed, 35 insertions(+), 20 deletions(-)
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/LocalProfile.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/LocalProfile.java
index 3c90ef5d9..da4cba0e2 100755
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/LocalProfile.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/LocalProfile.java
@@ -23,6 +23,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.stream.Stream;
import org.apache.inlong.agent.conf.JobProfile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,17 +50,21 @@ public class LocalProfile {
try {
if (Files.isDirectory(this.filePath)) {
// list parent path and find files which name is end with .json or .properties
- for (Iterator<Path> it = Files.list(this.filePath).iterator(); it.hasNext(); ) {
- String childPath = it.next().toString();
- JobProfile jobProfile = null;
- if (childPath.endsWith(JSON_SUFFIX)) {
- jobProfile = JobProfile.parseJsonFile(childPath);
- } else if (childPath.endsWith(PROPERTIES_SUFFIX)) {
- jobProfile = JobProfile.parsePropertiesFile(childPath);
- }
- if (jobProfile != null && jobProfile.allRequiredKeyExist()) {
- profileList.add(jobProfile);
+ try (final Stream<Path> pathStream = Files.list(this.filePath)) {
+ for (Iterator<Path> it = pathStream.iterator(); it.hasNext(); ) {
+ String childPath = it.next().toString();
+ JobProfile jobProfile = null;
+ if (childPath.endsWith(JSON_SUFFIX)) {
+ jobProfile = JobProfile.parseJsonFile(childPath);
+ } else if (childPath.endsWith(PROPERTIES_SUFFIX)) {
+ jobProfile = JobProfile.parsePropertiesFile(childPath);
+ }
+ if (jobProfile != null && jobProfile.allRequiredKeyExist()) {
+ profileList.add(jobProfile);
+ }
}
+ } catch (Exception e) {
+ LOGGER.error("error caught", e);
}
}
} catch (Exception ex) {
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/DirectoryTrigger.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/DirectoryTrigger.java
index 27eaf7d07..825c43fd0 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/DirectoryTrigger.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/DirectoryTrigger.java
@@ -159,9 +159,13 @@ public class DirectoryTrigger extends AbstractDaemon implements Trigger {
LOGGER.info("overflow got {}", parentPath);
// check whether parent path is valid.
if (Files.isDirectory(parentPath)) {
- for (Iterator<Path> it = Files.list(parentPath).iterator(); it.hasNext();) {
- Path childPath = it.next();
- registerAllSubDir(entity, parentPath.resolve(childPath), tmpWatchers);
+ try (final Stream<Path> pathStream = Files.list(parentPath)) {
+ for (Iterator<Path> it = pathStream.iterator(); it.hasNext(); ) {
+ Path childPath = it.next();
+ registerAllSubDir(entity, parentPath.resolve(childPath), tmpWatchers);
+ }
+ } catch (Exception e) {
+ LOGGER.error("error caught", e);
}
}
}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/PathPattern.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/PathPattern.java
index ef373fb9c..96c76423d 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/PathPattern.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/trigger/PathPattern.java
@@ -25,6 +25,8 @@ import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
+import java.util.stream.Stream;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.inlong.agent.plugin.filter.DateFormatRegex;
@@ -91,13 +93,17 @@ public class PathPattern {
if (dirPath.isFile() && dateFormatRegex.withFile(dirPath).match()) {
collectResult.add(dirPath);
} else if (dirPath.isDirectory()) {
- Files.list(dirPath.toPath()).forEach(path -> {
- try {
- walkAllSuitableFiles(path.toFile(), collectResult, maxNum);
- } catch (IOException ex) {
- LOGGER.warn("cannot add {}, please check it", path, ex);
- }
- });
+ try (final Stream<Path> pathStream = Files.list(dirPath.toPath())) {
+ pathStream.forEach(path -> {
+ try {
+ walkAllSuitableFiles(path.toFile(), collectResult, maxNum);
+ } catch (IOException ex) {
+ LOGGER.warn("cannot add {}, please check it", path, ex);
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error("error caught", e);
+ }
}
}