You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by jl...@apache.org on 2013/12/17 17:28:54 UTC
svn commit: r1551608 - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/
ha...
Author: jlowe
Date: Tue Dec 17 16:28:54 2013
New Revision: 1551608
URL: http://svn.apache.org/r1551608
Log:
svn merge -c 1551326 FIXES: YARN-1145. Fixed a potential file-handle leak in the web interface for displaying aggregated logs. Contributed by Rohith Sharma.
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Tue Dec 17 16:28:54 2013
@@ -15,6 +15,9 @@ Release 0.23.11 - UNRELEASED
YARN-1053. Diagnostic message from ContainerExitEvent is ignored in
ContainerImpl (Omkar Vinit Joshi via jlowe)
+ YARN-1145. Fixed a potential file-handle leak in the web interface for
+ displaying aggregated logs. (Rohith Sharma via jlowe)
+
Release 0.23.10 - 2013-12-09
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java Tue Dec 17 16:28:54 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.file.tfile.TFile;
import org.apache.hadoop.security.UserGroupInformation;
@@ -267,7 +268,7 @@ public class AggregatedLogFormat {
out.close();
}
- public void closeWriter() {
+ public void close() {
try {
this.writer.close();
} catch (IOException e) {
@@ -539,9 +540,8 @@ public class AggregatedLogFormat {
out.println("");
}
- public void close() throws IOException {
- this.scanner.close();
- this.fsDataIStream.close();
+ public void close() {
+ IOUtils.cleanup(LOG, scanner, reader, fsDataIStream);
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java Tue Dec 17 16:28:54 2013
@@ -57,109 +57,113 @@ public class AggregatedLogsBlock extends
@Override
protected void render(Block html) {
- ContainerId containerId = verifyAndGetContainerId(html);
- NodeId nodeId = verifyAndGetNodeId(html);
- String appOwner = verifyAndGetAppOwner(html);
- LogLimits logLimits = verifyAndGetLogLimits(html);
- if (containerId == null || nodeId == null || appOwner == null
- || appOwner.isEmpty() || logLimits == null) {
- return;
- }
-
- ApplicationId applicationId =
- containerId.getApplicationAttemptId().getApplicationId();
- String logEntity = $(ENTITY_STRING);
- if (logEntity == null || logEntity.isEmpty()) {
- logEntity = containerId.toString();
- }
-
- if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
- YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
- html.h1()
- ._("Aggregation is not enabled. Try the nodemanager at " + nodeId)
- ._();
- return;
- }
-
- Path remoteRootLogDir =
- new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
- YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
AggregatedLogFormat.LogReader reader = null;
try {
- reader =
- new AggregatedLogFormat.LogReader(conf,
- LogAggregationUtils.getRemoteNodeLogFileForApp(
- remoteRootLogDir, applicationId, appOwner, nodeId,
- LogAggregationUtils.getRemoteNodeLogDirSuffix(conf)));
- } catch (FileNotFoundException e) {
- // ACLs not available till the log file is opened.
- html.h1()
- ._("Logs not available for "
- + logEntity
- + ". Aggregation may not be complete, "
- + "Check back later or try the nodemanager at "
- + nodeId)._();
- return;
- } catch (IOException e) {
- html.h1()._("Error getting logs for " + logEntity)._();
- LOG.error("Error getting logs for " + logEntity, e);
- return;
- }
+ ContainerId containerId = verifyAndGetContainerId(html);
+ NodeId nodeId = verifyAndGetNodeId(html);
+ String appOwner = verifyAndGetAppOwner(html);
+ LogLimits logLimits = verifyAndGetLogLimits(html);
+ if (containerId == null || nodeId == null || appOwner == null
+ || appOwner.isEmpty() || logLimits == null) {
+ return;
+ }
- String owner = null;
- Map<ApplicationAccessType, String> appAcls = null;
- try {
- owner = reader.getApplicationOwner();
- appAcls = reader.getApplicationAcls();
- } catch (IOException e) {
- html.h1()._("Error getting logs for " + logEntity)._();
- LOG.error("Error getting logs for " + logEntity, e);
- return;
- }
- ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
- aclsManager.addApplication(applicationId, appAcls);
+ ApplicationId applicationId = containerId.getApplicationAttemptId()
+ .getApplicationId();
+ String logEntity = $(ENTITY_STRING);
+ if (logEntity == null || logEntity.isEmpty()) {
+ logEntity = containerId.toString();
+ }
- String remoteUser = request().getRemoteUser();
- UserGroupInformation callerUGI = null;
- if (remoteUser != null) {
- callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
- }
- if (callerUGI != null
- && !aclsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP,
- owner, applicationId)) {
- html.h1()
- ._("User [" + remoteUser
- + "] is not authorized to view the logs for " + logEntity)._();
- return;
- }
+ if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
+ html.h1()
+ ._("Aggregation is not enabled. Try the nodemanager at " + nodeId)
+ ._();
+ return;
+ }
- String desiredLogType = $(CONTAINER_LOG_TYPE);
- try {
- AggregatedLogFormat.ContainerLogsReader logReader =
- reader.getContainerLogsReader(containerId);
- if (logReader == null) {
- html.h1()._(
- "Logs not available for " + logEntity
- + ". Could be caused by the rentention policy")._();
+ Path remoteRootLogDir = new Path(conf.get(
+ YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
+
+ try {
+ reader = new AggregatedLogFormat.LogReader(conf,
+ LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir,
+ applicationId, appOwner, nodeId,
+ LogAggregationUtils.getRemoteNodeLogDirSuffix(conf)));
+ } catch (FileNotFoundException e) {
+ // ACLs not available till the log file is opened.
+ html.h1()
+ ._("Logs not available for " + logEntity
+ + ". Aggregation may not be complete, "
+ + "Check back later or try the nodemanager at " + nodeId)._();
+ return;
+ } catch (IOException e) {
+ html.h1()._("Error getting logs for " + logEntity)._();
+ LOG.error("Error getting logs for " + logEntity, e);
return;
}
- boolean foundLog = readContainerLogs(html, logReader, logLimits,
- desiredLogType);
+ String owner = null;
+ Map<ApplicationAccessType, String> appAcls = null;
+ try {
+ owner = reader.getApplicationOwner();
+ appAcls = reader.getApplicationAcls();
+ } catch (IOException e) {
+ html.h1()._("Error getting logs for " + logEntity)._();
+ LOG.error("Error getting logs for " + logEntity, e);
+ return;
+ }
+ ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
+ aclsManager.addApplication(applicationId, appAcls);
- if (!foundLog) {
- if (desiredLogType.isEmpty()) {
- html.h1("No logs available for container " + containerId.toString());
- } else {
- html.h1("Unable to locate '" + desiredLogType
- + "' log for container " + containerId.toString());
+ String remoteUser = request().getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ if (callerUGI != null
+ && !aclsManager.checkAccess(callerUGI,
+ ApplicationAccessType.VIEW_APP, owner, applicationId)) {
+ html.h1()
+ ._("User [" + remoteUser
+ + "] is not authorized to view the logs for " + logEntity)._();
+ return;
+ }
+
+ String desiredLogType = $(CONTAINER_LOG_TYPE);
+ try {
+ AggregatedLogFormat.ContainerLogsReader logReader = reader
+ .getContainerLogsReader(containerId);
+ if (logReader == null) {
+ html.h1()
+ ._("Logs not available for " + logEntity
+ + ". Could be caused by the rentention policy")._();
+ return;
+ }
+
+ boolean foundLog = readContainerLogs(html, logReader, logLimits,
+ desiredLogType);
+
+ if (!foundLog) {
+ if (desiredLogType.isEmpty()) {
+ html.h1("No logs available for container " + containerId.toString());
+ } else {
+ html.h1("Unable to locate '" + desiredLogType
+ + "' log for container " + containerId.toString());
+ }
+ return;
}
+ } catch (IOException e) {
+ html.h1()._("Error getting logs for " + logEntity)._();
+ LOG.error("Error getting logs for " + logEntity, e);
return;
}
- } catch (IOException e) {
- html.h1()._("Error getting logs for " + logEntity)._();
- LOG.error("Error getting logs for " + logEntity, e);
- return;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java Tue Dec 17 16:28:54 2013
@@ -100,7 +100,7 @@ public class TestAggregatedLogFormat {
testContainerId);
logWriter.append(logKey, logValue);
- logWriter.closeWriter();
+ logWriter.close();
// make sure permission are correct on the file
FileStatus fsStatus = fs.getFileStatus(remoteAppLogFile);
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java Tue Dec 17 16:28:54 2013
@@ -233,7 +233,7 @@ public class TestAggregatedLogsBlock {
writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"),
new AggregatedLogFormat.LogValue(rootLogDirs, containerId));
- writer.closeWriter();
+ writer.close();
}
private void writeLogs(String dirName) throws Exception {
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1551608&r1=1551607&r2=1551608&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java Tue Dec 17 16:28:54 2013
@@ -176,7 +176,7 @@ public class AppLogAggregatorImpl implem
localAppLogDirs);
if (this.writer != null) {
- this.writer.closeWriter();
+ this.writer.close();
LOG.info("Finished aggregate log-file for app " + this.applicationId);
}