You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/07/31 12:56:02 UTC
[hadoop] branch branch-3.1 updated: YARN-6221. Entities missing
from ATS when summary log file info got returned to the ATS before the
domain log. Contributed by Xiaomin Zhang
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ac30549 YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
ac30549 is described below
commit ac30549b06866dfab61eb86169841d4cdbf50d66
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Sat Jul 31 14:55:43 2021 +0200
YARN-6221. Entities missing from ATS when summary log file info got returned to the ATS before the domain log. Contributed by Xiaomin Zhang
---
.../timeline/EntityGroupFSTimelineStore.java | 3 ++-
.../timeline/TestEntityGroupFSTimelineStore.java | 25 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index 80baf89..fd8c57f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -780,10 +780,11 @@ public class EntityGroupFSTimelineStore extends CompositeService
LogInfo log;
if (isDomainLog) {
log = new DomainLogInfo(attemptDirName, filename, owner);
+ summaryLogs.add(0, log);
} else {
log = new EntityLogInfo(attemptDirName, filename, owner);
+ summaryLogs.add(log);
}
- summaryLogs.add(log);
}
private synchronized void addDetailLog(String attemptDirName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index 61da3c8..13224d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -225,6 +225,31 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
}
@Test
+ public void testAppLogsDomainLogLastlyScanned() throws Exception {
+ EntityGroupFSTimelineStore.AppLogs appLogs =
+ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+ AppState.COMPLETED);
+ Path attemptDirPath = new Path(new Path(testActiveDirPath,
+ mainTestAppId.toString()),
+ getAttemptDirName(mainTestAppId));
+ //Delete the domain log from AppDirPath so first scan won't find it
+ fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false);
+ appLogs.scanForLogs();
+ List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
+ assertEquals(1, summaryLogs.size());
+ assertEquals(TEST_SUMMARY_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+
+ //Generate the domain log
+ FSDataOutputStream out = fs.create(
+ new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
+ out.close();
+
+ appLogs.scanForLogs();
+ assertEquals(2, summaryLogs.size());
+ assertEquals(TEST_DOMAIN_LOG_FILE_NAME, summaryLogs.get(0).getFilename());
+ }
+
+ @Test
public void testMoveToDone() throws Exception {
EntityGroupFSTimelineStore.AppLogs appLogs =
store.new AppLogs(mainTestAppId, mainTestAppDirPath,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org