[dolphinscheduler] branch dev updated: fix work oom when task logs's size is very large (#11224)

[ke...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kerwin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 9a53c6ac34 fix work oom when task logs's size is very large (#11224)
9a53c6ac34 is described below

commit 9a53c6ac3430c9edb1ded218d5a0190214fadc80
Author: microeastcowboy <mi...@gmail.com>
AuthorDate: Mon Aug 1 15:28:06 2022 +0800

    fix work oom when task logs's size is very large (#11224)
---
 .../plugin/task/api/AbstractCommandExecutor.java   | 45 +++++++---------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
index 369e411d9d..aa169676ed 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
@@ -404,48 +404,31 @@ public abstract class AbstractCommandExecutor {
      * @return app id list
      */
     private List<String> getAppIds(String logPath) {
-        List<String> logs = convertFile2List(logPath);
-
         List<String> appIds = new ArrayList<>();
-        /*
-         * analysis log?get submited yarn application id
-         */
-        for (String log : logs) {
-            String appId = findAppId(log);
-            if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) {
-                logger.info("find app id: {}", appId);
-                appIds.add(appId);
-            }
-        }
-        return appIds;
-    }
-
-    /**
-     * convert file to list
-     *
-     * @param filename file name
-     * @return line list
-     */
-    private List<String> convertFile2List(String filename) {
-        List<String> lineList = new ArrayList<>(100);
-        File file = new File(filename);
 
+        File file = new File(logPath);
         if (!file.exists()) {
-            return lineList;
+            return appIds;
         }
 
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), StandardCharsets.UTF_8))) {
+        /*
+         * analysis log?get submited yarn application id
+         */
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logPath), StandardCharsets.UTF_8))) {
             String line;
             while ((line = br.readLine()) != null) {
-                lineList.add(line);
+                String appId = findAppId(line);
+                if (StringUtils.isNotEmpty(appId) && !appIds.contains(appId)) {
+                    logger.info("find app id: {}", appId);
+                    appIds.add(appId);
+                }
             }
         } catch (Exception e) {
-            logger.error(String.format("read file: %s failed : ", filename), e);
+            logger.error(String.format("read file: %s failed : ", logPath), e);
         }
-
-        return lineList;
+        return appIds;
     }
-    
+
     /**
      * find var pool
      * @param line