You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/03/13 02:22:38 UTC
[inlong] branch master updated: [INLONG-7533][Agent] Fix that log cannot be collected for position reset (#7538)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new b6014fa5f [INLONG-7533][Agent] Fix that log cannot be collected for position reset (#7538)
b6014fa5f is described below
commit b6014fa5f4af5ad4bf4196c0f26d0ab104b74db1
Author: doleyzi <43...@users.noreply.github.com>
AuthorDate: Mon Mar 13 10:22:31 2023 +0800
[INLONG-7533][Agent] Fix that log cannot be collected for position reset (#7538)
Co-authored-by: doleyzi <do...@tencent.com>
Co-authored-by: healchow <he...@gmail.com>
---
.../sources/reader/file/FileReaderOperator.java | 5 +--
.../sources/reader/file/MonitorTextFile.java | 37 ++++++++++++++++++++--
inlong-agent/conf/log4j2.xml | 6 ++--
3 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/FileReaderOperator.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/FileReaderOperator.java
index 583632877..61610d3c9 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/FileReaderOperator.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/FileReaderOperator.java
@@ -89,7 +89,7 @@ public class FileReaderOperator extends AbstractReader {
public File file;
public int position = 0;
- private int bytePosition = 0;
+ public int bytePosition = 0;
private long readEndpoint = Long.MAX_VALUE;
public String md5;
public Map<String, String> metadata;
@@ -97,10 +97,11 @@ public class FileReaderOperator extends AbstractReader {
public boolean inited = false;
public volatile boolean finished = false;
public String instanceId;
+ public String fileKey = null;
private long timeout;
private long waitTimeout;
private long lastTime = 0;
- private List<Validator> validators = new ArrayList<>();
+ private final List<Validator> validators = new ArrayList<>();
private final BlockingQueue<String> queue = new LinkedBlockingQueue<>(CACHE_QUEUE_SIZE);
private final StringBuffer sb = new StringBuffer();
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/MonitorTextFile.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/MonitorTextFile.java
index e5cb5206a..c0ad4f49c 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/MonitorTextFile.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/MonitorTextFile.java
@@ -41,7 +41,9 @@ import static org.apache.inlong.agent.constant.JobConstants.JOB_FILE_MONITOR_INT
public final class MonitorTextFile {
private static final Logger LOGGER = LoggerFactory.getLogger(MonitorTextFile.class);
- // monitor thread pool
+ /**
+ * monitor thread pool
+ */
private static final ThreadPoolExecutor EXECUTOR_SERVICE = new ThreadPoolExecutor(
0, Integer.MAX_VALUE,
60L, TimeUnit.SECONDS,
@@ -131,16 +133,22 @@ public final class MonitorTextFile {
try {
attributesAfter = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
currentPath = file.getCanonicalPath();
+
+ // Determine whether the inode has changed
+ if (isInodeChanged(attributesAfter.fileKey().toString())) {
+ resetPosition();
+ }
+ fileReaderOperator.fileKey = attributesAfter.fileKey().toString();
} catch (Exception e) {
// set position 0 when split file
- fileReaderOperator.position = 0;
+ resetPosition();
LOGGER.error(String.format("monitor file %s error, reset position to 0", file.getName()), e);
return;
}
// if change symbolic links
if (attributesAfter.isSymbolicLink() && !path.equals(currentPath)) {
- fileReaderOperator.position = 0;
+ resetPosition();
path = currentPath;
}
@@ -148,5 +156,28 @@ public final class MonitorTextFile {
fileReaderOperator.fetchData();
}
}
+
+ /**
+ * Reset the position and bytePosition
+ */
+ private void resetPosition() {
+ LOGGER.info("reset position {}", fileReaderOperator.file.toPath());
+ fileReaderOperator.position = 0;
+ fileReaderOperator.bytePosition = 0;
+ }
+
+ /**
+ * Determine whether the inode has changed
+ *
+ * @param currentFileKey current file key
+ * @return true if the inode changed, otherwise false
+ */
+ private boolean isInodeChanged(String currentFileKey) {
+ if (fileReaderOperator.fileKey == null || currentFileKey == null) {
+ return false;
+ }
+
+ return !fileReaderOperator.fileKey.equals(currentFileKey);
+ }
}
}
diff --git a/inlong-agent/conf/log4j2.xml b/inlong-agent/conf/log4j2.xml
index e01582ffe..66165f4f3 100644
--- a/inlong-agent/conf/log4j2.xml
+++ b/inlong-agent/conf/log4j2.xml
@@ -23,8 +23,8 @@
<property name="log_pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} -%5p ${PID:-} [%30.30t] %-30.30C{1.}:%L %m%n</property>
<property name="every_file_date">1</property>
<property name="every_file_size">1G</property>
- <property name="output_log_level">DEBUG</property>
- <property name="rolling_max">50</property>
+ <property name="output_log_level">INFO</property>
+ <property name="rolling_max">15</property>
<property name="debug_fileName">${basePath}/debug.log</property>
<property name="debug_filePattern">${basePath}/debug-%d{yyyy-MM-dd}-%i.log.gz</property>
<property name="debug_max">10</property>
@@ -37,7 +37,7 @@
<property name="error_fileName">${basePath}/error.log</property>
<property name="error_filePattern">${basePath}/error-%d{yyyy-MM-dd}-%i.log.gz</property>
<property name="error_max">10</property>
- <property name="console_print_level">DEBUG</property>
+ <property name="console_print_level">INFO</property>
<property name="index_max">10</property>
<property name="index_fileName">${basePath}/index.log</property>
<property name="index_filePattern">${basePath}/index-%d{yyyy-MM-dd}-%i.log.gz</property>