You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/07/27 14:49:15 UTC

[hadoop] branch trunk updated: YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang

This is an automated email from the ASF dual-hosted git repository.

snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new fa0289b  YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
fa0289b is described below

commit fa0289b0226f40fa5c7f898de586b8fd8b763374
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Jul 27 16:47:41 2021 +0200

    YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
---
 .../timeline/EntityGroupFSTimelineStore.java       |  3 ++-
 .../timeline/TestEntityGroupFSTimelineStore.java   | 25 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index e9957c8..c2992b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -808,10 +808,11 @@ public class EntityGroupFSTimelineStore extends CompositeService
       LogInfo log;
       if (isDomainLog) {
         log = new DomainLogInfo(attemptDirName, filename, owner);
+        summaryLogs.add(0, log);
       } else {
         log = new EntityLogInfo(attemptDirName, filename, owner);
+        summaryLogs.add(log);
       }
-      summaryLogs.add(log);
     }
 
     private synchronized void addDetailLog(String attemptDirName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index 984e157..7302ae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -227,6 +227,31 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
   }
 
   @Test
+  public void testAppLogsDomainLogLastlyScanned() throws Exception {
+    EntityGroupFSTimelineStore.AppLogs appLogs =
+            store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+                    AppState.COMPLETED);
+    Path attemptDirPath = new Path(new Path(testActiveDirPath,
+            mainTestAppId.toString()),
+            getAttemptDirName(mainTestAppId));
+    //Delete the domain log from AppDirPath so first scan won't find it
+    fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false);
+    appLogs.scanForLogs();
+    List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
+    assertEquals(1, summaryLogs.size());
+    assertEquals(TEST_SUMMARY_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+
+    //Generate the domain log
+    FSDataOutputStream out = fs.create(
+            new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
+    out.close();
+
+    appLogs.scanForLogs();
+    assertEquals(2, summaryLogs.size());
+    assertEquals(TEST_DOMAIN_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+  }
+
+  @Test
   public void testMoveToDone() throws Exception {
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org