You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/08/29 20:16:58 UTC
hadoop git commit: YARN-7083. Log aggregation deletes/renames while
file is open. Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 fe4aff733 -> 79294b5f3
YARN-7083. Log aggregation deletes/renames while file is open. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79294b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79294b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79294b5f
Branch: refs/heads/branch-2.8
Commit: 79294b5f329317c30775f8b4dc61440f4b6c24cd
Parents: fe4aff7
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 29 13:18:49 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue Aug 29 13:18:49 2017 -0700
----------------------------------------------------------------------
.../logaggregation/AppLogAggregatorImpl.java | 83 ++++++++++----------
1 file changed, 42 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79294b5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index df972f0..e2975b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -320,53 +320,54 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
logAggregationTimes++;
String diagnosticMessage = "";
boolean logAggregationSucceedInThisCycle = true;
- try (LogWriter writer = new LogWriter()){
- try {
- writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp,
- this.userUgi);
- // Write ACLs once when the writer is created.
- writer.writeApplicationACLs(appAcls);
- writer.writeApplicationOwner(this.userUgi.getShortUserName());
- } catch (IOException e1) {
- logAggregationSucceedInThisCycle = false;
- LOG.error("Cannot create writer for app " + this.applicationId
- + ". Skip log upload this time. ", e1);
- return;
- }
-
+ try {
boolean uploadedLogsInThisCycle = false;
- for (ContainerId container : pendingContainerInThisCycle) {
- ContainerLogAggregator aggregator = null;
- if (containerLogAggregators.containsKey(container)) {
- aggregator = containerLogAggregators.get(container);
- } else {
- aggregator = new ContainerLogAggregator(container);
- containerLogAggregators.put(container, aggregator);
- }
- Set<Path> uploadedFilePathsInThisCycle =
- aggregator.doContainerLogAggregation(writer, appFinished,
- finishedContainers.contains(container));
- if (uploadedFilePathsInThisCycle.size() > 0) {
- uploadedLogsInThisCycle = true;
- this.delService.delete(this.userUgi.getShortUserName(), null,
- uploadedFilePathsInThisCycle
- .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
+ try (LogWriter writer = new LogWriter()){
+ try {
+ writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp,
+ this.userUgi);
+ // Write ACLs once when the writer is created.
+ writer.writeApplicationACLs(appAcls);
+ writer.writeApplicationOwner(this.userUgi.getShortUserName());
+ } catch (IOException e1) {
+ logAggregationSucceedInThisCycle = false;
+ LOG.error("Cannot create writer for app " + this.applicationId
+ + ". Skip log upload this time. ", e1);
+ return;
}
- // This container is finished, and all its logs have been uploaded,
- // remove it from containerLogAggregators.
- if (finishedContainers.contains(container)) {
- containerLogAggregators.remove(container);
+ for (ContainerId container : pendingContainerInThisCycle) {
+ ContainerLogAggregator aggregator = null;
+ if (containerLogAggregators.containsKey(container)) {
+ aggregator = containerLogAggregators.get(container);
+ } else {
+ aggregator = new ContainerLogAggregator(container);
+ containerLogAggregators.put(container, aggregator);
+ }
+ Set<Path> uploadedFilePathsInThisCycle =
+ aggregator.doContainerLogAggregation(writer, appFinished,
+ finishedContainers.contains(container));
+ if (uploadedFilePathsInThisCycle.size() > 0) {
+ uploadedLogsInThisCycle = true;
+ this.delService.delete(this.userUgi.getShortUserName(), null,
+ uploadedFilePathsInThisCycle
+ .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
+ }
+
+ // This container is finished, and all its logs have been uploaded,
+ // remove it from containerLogAggregators.
+ if (finishedContainers.contains(container)) {
+ containerLogAggregators.remove(container);
+ }
}
- }
- // Before upload logs, make sure the number of existing logs
- // is smaller than the configured NM log aggregation retention size.
- if (uploadedLogsInThisCycle && logAggregationInRolling) {
- cleanOldLogs();
- cleanupOldLogTimes++;
+ // Before upload logs, make sure the number of existing logs
+ // is smaller than the configured NM log aggregation retention size.
+ if (uploadedLogsInThisCycle && logAggregationInRolling) {
+ cleanOldLogs();
+ cleanupOldLogTimes++;
+ }
}
-
long currentTime = System.currentTimeMillis();
final Path renamedPath = this.rollingMonitorInterval <= 0
? remoteNodeLogFileForApp : new Path(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org