You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/07/31 12:53:54 UTC
[hadoop] branch branch-3.3 updated: YARN-6221. Entities missing
from ATS when summary log file info got returned to the ATS before the
domain log. Contributed by Xiaomin Zhang
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 0a0628b YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
0a0628b is described below
commit 0a0628b1cbaab3fb0a1c71781b77dd2c91ed0c73
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Sat Jul 31 14:53:30 2021 +0200
YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
---
.../timeline/EntityGroupFSTimelineStore.java | 3 ++-
.../timeline/TestEntityGroupFSTimelineStore.java | 25 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index e9957c8..c2992b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -808,10 +808,11 @@ public class EntityGroupFSTimelineStore extends CompositeService
LogInfo log;
if (isDomainLog) {
log = new DomainLogInfo(attemptDirName, filename, owner);
+ summaryLogs.add(0, log);
} else {
log = new EntityLogInfo(attemptDirName, filename, owner);
+ summaryLogs.add(log);
}
- summaryLogs.add(log);
}
private synchronized void addDetailLog(String attemptDirName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index 984e157..7302ae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -227,6 +227,31 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
}
@Test
+ public void testAppLogsDomainLogLastlyScanned() throws Exception {
+ EntityGroupFSTimelineStore.AppLogs appLogs =
+ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+ AppState.COMPLETED);
+ Path attemptDirPath = new Path(new Path(testActiveDirPath,
+ mainTestAppId.toString()),
+ getAttemptDirName(mainTestAppId));
+ //Delete the domain log from AppDirPath so first scan won't find it
+ fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false);
+ appLogs.scanForLogs();
+ List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
+ assertEquals(1, summaryLogs.size());
+ assertEquals(TEST_SUMMARY_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+
+ //Generate the domain log
+ FSDataOutputStream out = fs.create(
+ new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
+ out.close();
+
+ appLogs.scanForLogs();
+ assertEquals(2, summaryLogs.size());
+ assertEquals(TEST_DOMAIN_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+ }
+
+ @Test
public void testMoveToDone() throws Exception {
EntityGroupFSTimelineStore.AppLogs appLogs =
store.new AppLogs(mainTestAppId, mainTestAppDirPath,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org