You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/05/10 05:47:38 UTC
hadoop git commit: YARN-4842. Fixed "yarn logs" command to guess (and
thus not require) the appOwner argument when viewing another user's logs.
Contributed by Ram Venkatesh and Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk 996a210ab -> 87f5e3513
YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f5e351
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f5e351
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f5e351
Branch: refs/heads/trunk
Commit: 87f5e351337a905af5215af76c72b9312616cd4f
Parents: 996a210
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Mon May 9 22:33:14 2016 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon May 9 22:41:18 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 55 +++--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 203 ++++++++++++++++---
.../yarn/logaggregation/LogCLIHelpers.java | 82 +++++++-
3 files changed, 290 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 487b694..926ba79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -151,14 +151,12 @@ public class LogsCLI extends Configured implements Tool {
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
- if (appOwner == null || appOwner.isEmpty()) {
- appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
- }
-
boolean appStateObtainedSuccessfully = true;
YarnApplicationState appState = YarnApplicationState.NEW;
+ ApplicationReport appReport = null;
try {
- appState = getApplicationState(appId);
+ appReport = getApplicationReport(appId);
+ appState = appReport.getYarnApplicationState();
if (appState == YarnApplicationState.NEW
|| appState == YarnApplicationState.NEW_SAVING
|| appState == YarnApplicationState.SUBMITTED) {
@@ -171,6 +169,16 @@ public class LogsCLI extends Configured implements Tool {
+ " Attempting to fetch logs directly from the filesystem.");
}
+ if (appOwner == null || appOwner.isEmpty()) {
+ appOwner = guessAppOwner(appReport, appId);
+ if (appOwner == null) {
+ System.err.println("Can not find the appOwner. "
+ + "Please specify the correct appOwner");
+ System.err.println("Could not locate application logs for " + appId);
+ return -1;
+ }
+ }
+
if (showMetaInfo) {
return showMetaInfo(appState, appStateObtainedSuccessfully,
logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
@@ -201,6 +209,10 @@ public class LogsCLI extends Configured implements Tool {
if (nodeAddress == null) {
resultCode =
logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
+ if (resultCode == -1) {
+ System.err.println("Can not find the logs for the application: "
+ + appId + " with the appOwner: " + appOwner);
+ }
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
@@ -210,13 +222,12 @@ public class LogsCLI extends Configured implements Tool {
return resultCode;
}
- private YarnApplicationState getApplicationState(ApplicationId appId)
+ private ApplicationReport getApplicationReport(ApplicationId appId)
throws IOException, YarnException {
YarnClient yarnClient = createYarnClient();
try {
- ApplicationReport appReport = yarnClient.getApplicationReport(appId);
- return appReport.getYarnApplicationState();
+ return yarnClient.getApplicationReport(appId);
} finally {
yarnClient.close();
}
@@ -693,11 +704,12 @@ public class LogsCLI extends Configured implements Tool {
amContainersList, logFiles, logCliHelper, appOwner, true);
} else {
System.err.println("Can not get AMContainers logs for "
- + "the application:" + appId);
- System.err.println("This application:" + appId + " is finished."
- + " Please enable the application history service. Or Using "
- + "yarn logs -applicationId <appId> -containerId <containerId> "
- + "--nodeAddress <nodeHttpAddress> to get the container logs");
+ + "the application:" + appId + " with the appOwner:" + appOwner);
+ System.err.println("This application:" + appId + " has finished."
+ + " Please enable the application-history service or explicitly"
+ + " use 'yarn logs -applicationId <appId> "
+ + "-containerId <containerId> --nodeAddress <nodeHttpAddress>' "
+ + "to get the container logs.");
return -1;
}
}
@@ -750,7 +762,8 @@ public class LogsCLI extends Configured implements Tool {
appOwner);
} else if (!isApplicationFinished(appState)) {
System.err.println("Unable to get logs for this container:"
- + containerIdStr + "for the application:" + appIdStr);
+ + containerIdStr + "for the application:" + appIdStr
+ + " with the appOwner: " + appOwner);
System.err.println("The application: " + appIdStr
+ " is still running, and we can not get Container report "
+ "for the container: " + containerIdStr +". Please try later "
@@ -821,4 +834,18 @@ public class LogsCLI extends Configured implements Tool {
return isAppFinished;
}
}
+
+ private String guessAppOwner(ApplicationReport appReport,
+ ApplicationId appId) throws IOException {
+ String appOwner = null;
+ if (appReport != null) {
+ //always use the app owner from the app report if possible
+ appOwner = appReport.getUser();
+ } else {
+ appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+ appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
+ appId, appOwner, getConf());
+ }
+ return appOwner;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index b950764..4f1c629 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -42,11 +42,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -55,22 +55,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class TestLogsCLI {
+
ByteArrayOutputStream sysOutStream;
private PrintStream sysOut;
-
+
ByteArrayOutputStream sysErrStream;
private PrintStream sysErr;
@@ -79,7 +78,7 @@ public class TestLogsCLI {
sysOutStream = new ByteArrayOutputStream();
sysOut = new PrintStream(sysOutStream);
System.setOut(sysOut);
-
+
sysErrStream = new ByteArrayOutputStream();
sysErr = new PrintStream(sysErrStream);
System.setErr(sysErr);
@@ -91,16 +90,18 @@ public class TestLogsCLI {
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
LogCLIHelpers cliHelper = new LogCLIHelpers();
cliHelper.setConf(conf);
- YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+ YarnClient mockYarnClient = createMockYarnClient(
+ YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
dumper.setConf(conf);
-
+
// verify dumping a non-existent application's logs returns a failure code
int exitCode = dumper.run( new String[] {
"-applicationId", "application_0_0" } );
assertTrue("Should return an error code", exitCode != 0);
-
- // verify dumping a non-existent container log is a failure code
+
+ // verify dumping a non-existent container log is a failure code
exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
"nonexistentnode:1234", "nobody");
assertTrue("Should return an error code", exitCode != 0);
@@ -109,10 +110,12 @@ public class TestLogsCLI {
@Test(timeout = 5000l)
public void testInvalidApplicationId() throws Exception {
Configuration conf = new YarnConfiguration();
- YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+ YarnClient mockYarnClient = createMockYarnClient(
+ YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(conf);
-
+
int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"});
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
@@ -137,7 +140,9 @@ public class TestLogsCLI {
@Test(timeout = 5000l)
public void testHelpMessage() throws Exception {
Configuration conf = new YarnConfiguration();
- YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+ YarnClient mockYarnClient = createMockYarnClient(
+ YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
dumper.setConf(conf);
@@ -187,7 +192,7 @@ public class TestLogsCLI {
String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString());
}
-
+
@Test (timeout = 15000)
public void testFetchApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/";
@@ -200,13 +205,13 @@ public class TestLogsCLI {
FileSystem fs = FileSystem.get(configuration);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
- ApplicationAttemptIdPBImpl.newInstance(appId, 1);
- ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0);
- ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
- ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2);
- ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3);
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
+ ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
+ ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
+ ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
NodeId nodeId = NodeId.newInstance("localhost", 1234);
// create local logs
@@ -222,6 +227,7 @@ public class TestLogsCLI {
fs.delete(appLogsDir, true);
}
assertTrue(fs.mkdirs(appLogsDir));
+
List<String> rootLogDirs = Arrays.asList(rootLogDir);
List<String> logTypes = new ArrayList<String>();
@@ -258,7 +264,8 @@ public class TestLogsCLI {
containerId3, path, fs);
YarnClient mockYarnClient =
- createMockYarnClient(YarnApplicationState.FINISHED);
+ createMockYarnClient(
+ YarnApplicationState.FINISHED, ugi.getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
@@ -349,6 +356,143 @@ public class TestLogsCLI {
}
@Test (timeout = 15000)
+ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
+ String remoteLogRootDir = "target/logs/";
+ String rootLogDir = "target/LocalLogs";
+
+ String testUser = "test";
+ UserGroupInformation testUgi = UserGroupInformation
+ .createRemoteUser(testUser);
+
+ Configuration configuration = new Configuration();
+ configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+ configuration
+ .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
+ configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+ FileSystem fs = FileSystem.get(configuration);
+
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId
+ .newContainerId(appAttemptId, 1);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+
+ try {
+ Path rootLogDirPath = new Path(rootLogDir);
+ if (fs.exists(rootLogDirPath)) {
+ fs.delete(rootLogDirPath, true);
+ }
+ assertTrue(fs.mkdirs(rootLogDirPath));
+
+ // create local app dir for app
+ final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
+ if (fs.exists(appLogsDir)) {
+ fs.delete(appLogsDir, true);
+ }
+ assertTrue(fs.mkdirs(appLogsDir));
+
+ List<String> rootLogDirs = Arrays.asList(rootLogDir);
+ List<String> logTypes = new ArrayList<String>();
+ logTypes.add("syslog");
+
+ // create container logs in localLogDir for app
+ createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
+
+ // create the remote app dir for app
+ // but for a different user testUser"
+ Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ assertTrue(fs.mkdirs(path));
+
+ // upload container logs for app into remote dir
+ uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
+ nodeId, containerId, path, fs);
+
+ YarnClient mockYarnClient = createMockYarnClient(
+ YarnApplicationState.FINISHED, testUgi.getShortUserName());
+ LogsCLI cli = new LogsCLIForTest(mockYarnClient);
+ cli.setConf(configuration);
+
+ // Verify that we can get the application logs by specifying
+ // a correct appOwner
+ int exitCode = cli.run(new String[] {
+ "-applicationId", appId.toString(),
+ "-appOwner", testUser});
+ assertTrue(exitCode == 0);
+ assertTrue(sysOutStream.toString().contains(
+ "Hello " + containerId + " in syslog!"));
+ sysOutStream.reset();
+
+ // Verify that we can not get the application logs
+ // if an invalid user is specified
+ exitCode = cli.run(new String[] {
+ "-applicationId", appId.toString(),
+ "-appOwner", "invalid"});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains("Can not find the logs "
+ + "for the application: " + appId.toString()));
+ sysErrStream.reset();
+
+ // Verify that we do not specify appOwner, and can not
+ // get appReport from RM, we still can figure out the appOwner
+ // and can get app logs successfully.
+ YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
+ cli = new LogsCLIForTest(mockYarnClient2);
+ cli.setConf(configuration);
+ exitCode = cli.run(new String[] {
+ "-applicationId", appId.toString()});
+ assertTrue(exitCode == 0);
+ assertTrue(sysOutStream.toString().contains("Hello "
+ + containerId + " in syslog!"));
+ sysOutStream.reset();
+
+ // Verify that we could get the err message "Can not find the appOwner"
+ // if we do not specify the appOwner, can not get appReport, and
+ // the app does not exist in remote dir.
+ ApplicationId appId2 = ApplicationId.newInstance(
+ System.currentTimeMillis(), 2);
+ exitCode = cli.run(new String[] {
+ "-applicationId", appId2.toString()});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains(
+ "Can not find the appOwner"));
+ sysErrStream.reset();
+
+ // Verify that we could not get appOwner
+ // because we don't have file-system permissions
+ ApplicationId appTest = ApplicationId.newInstance(
+ System.currentTimeMillis(), 1000);
+ String priorityUser = "priority";
+ Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
+ + "/logs/" + appTest);
+ if (fs.exists(pathWithoutPerm)) {
+ fs.delete(pathWithoutPerm, true);
+ }
+ // The user will not have read permission for this directory.
+ // To mimic the scenario that the user can not get file status
+ FsPermission permission = FsPermission
+ .createImmutable((short) 01300);
+ assertTrue(fs.mkdirs(pathWithoutPerm, permission));
+
+ exitCode = cli.run(new String[] {
+ "-applicationId", appTest.toString()});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains(
+ "Guessed logs' owner is " + priorityUser + " and current user "
+ + UserGroupInformation.getCurrentUser().getUserName()
+ + " does not have permission to access"));
+ sysErrStream.reset();
+ } finally {
+ fs.delete(new Path(remoteLogRootDir), true);
+ fs.delete(new Path(rootLogDir), true);
+ }
+ }
+
+ @Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
@@ -380,7 +524,8 @@ public class TestLogsCLI {
appId, containerIds, nodeIds);
YarnClient mockYarnClient =
- createMockYarnClient(YarnApplicationState.FINISHED);
+ createMockYarnClient(YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
@@ -466,7 +611,8 @@ public class TestLogsCLI {
appId, containerIds, nodeIds);
YarnClient mockYarnClient =
- createMockYarnClient(YarnApplicationState.FINISHED);
+ createMockYarnClient(YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
@@ -508,7 +654,8 @@ public class TestLogsCLI {
assertTrue(fs.exists(harPath));
YarnClient mockYarnClient =
- createMockYarnClient(YarnApplicationState.FINISHED);
+ createMockYarnClient(YarnApplicationState.FINISHED,
+ ugi.getShortUserName());
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
int exitCode = cli.run(new String[]{"-applicationId",
@@ -630,10 +777,12 @@ public class TestLogsCLI {
writer.close();
}
- private YarnClient createMockYarnClient(YarnApplicationState appState)
+ private YarnClient createMockYarnClient(YarnApplicationState appState,
+ String user)
throws YarnException, IOException {
YarnClient mockClient = mock(YarnClient.class);
ApplicationReport mockAppReport = mock(ApplicationReport.class);
+ doReturn(user).when(mockAppReport).getUser();
doReturn(appState).when(mockAppReport).getYarnApplicationState();
doReturn(mockAppReport).when(mockClient).getApplicationReport(
any(ApplicationId.class));
@@ -659,9 +808,9 @@ public class TestLogsCLI {
}
private static class LogsCLIForTest extends LogsCLI {
-
+
private YarnClient yarnClient;
-
+
public LogsCLIForTest(YarnClient yarnClient) {
super();
this.yarnClient = yarnClient;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index e3ea1f6..ba0dd89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -23,6 +23,7 @@ import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
+import java.nio.file.AccessDeniedException;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.HarFs;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
@@ -56,6 +59,53 @@ public class LogCLIHelpers implements Configurable {
@Private
@VisibleForTesting
+ /**
+ * Return the owner for a given AppId
+ * @param remoteRootLogDir
+ * @param appId
+ * @param bestGuess
+ * @param conf
+ * @return the owner or null
+ * @throws IOException
+ */
+ public static String getOwnerForAppIdOrNull(
+ ApplicationId appId, String bestGuess,
+ Configuration conf) throws IOException {
+ Path remoteRootLogDir = new Path(conf.get(
+ YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
+ String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
+ Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
+ appId, bestGuess, suffix);
+ FileContext fc =
+ FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
+ String pathAccess = fullPath.toString();
+ try {
+ if (fc.util().exists(fullPath)) {
+ return bestGuess;
+ }
+ Path toMatch = LogAggregationUtils.
+ getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
+ pathAccess = toMatch.toString();
+ FileStatus[] matching = fc.util().globStatus(toMatch);
+ if (matching == null || matching.length != 1) {
+ return null;
+ }
+ //fetch the user from the full path /app-logs/user[/suffix]/app_id
+ Path parent = matching[0].getPath().getParent();
+ //skip the suffix too
+ if (suffix != null && !StringUtils.isEmpty(suffix)) {
+ parent = parent.getParent();
+ }
+ return parent.getName();
+ } catch (AccessControlException | AccessDeniedException ex) {
+ logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
+ return null;
+ }
+ }
+
+ @Private
+ @VisibleForTesting
public int dumpAContainersLogsForALogType(String appId, String containerId,
String nodeId, String jobOwner, List<String> logType)
throws IOException {
@@ -93,12 +143,12 @@ public class LogCLIHelpers implements Configurable {
thisNodeFile.getPath());
if (logType == null) {
if (dumpAContainerLogs(containerId, reader, System.out,
- thisNodeFile.getModificationTime()) > -1) {
+ thisNodeFile.getModificationTime()) > -1) {
foundContainerLogs = true;
}
} else {
if (dumpAContainerLogsForALogType(containerId, reader, System.out,
- thisNodeFile.getModificationTime(), logType) > -1) {
+ thisNodeFile.getModificationTime(), logType) > -1) {
foundContainerLogs = true;
}
}
@@ -182,7 +232,7 @@ public class LogCLIHelpers implements Configurable {
while (true) {
try {
LogReader.readAContainerLogsForALogType(valueStream, out,
- logUploadedTime);
+ logUploadedTime);
foundContainerLogs = true;
} catch (EOFException eof) {
break;
@@ -249,9 +299,10 @@ public class LogCLIHelpers implements Configurable {
continue;
}
if (!thisNodeFile.getPath().getName()
- .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
+ .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
AggregatedLogFormat.LogReader reader =
- new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
+ new AggregatedLogFormat.LogReader(getConf(),
+ thisNodeFile.getPath());
try {
DataInputStream valueStream;
@@ -261,13 +312,14 @@ public class LogCLIHelpers implements Configurable {
while (valueStream != null) {
String containerString =
- "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
+ "\n\nContainer: " + key + " on "
+ + thisNodeFile.getPath().getName();
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
try {
LogReader.readAContainerLogsForALogType(valueStream, out,
- thisNodeFile.getModificationTime());
+ thisNodeFile.getModificationTime());
foundAnyLogs = true;
} catch (EOFException eof) {
break;
@@ -283,7 +335,7 @@ public class LogCLIHelpers implements Configurable {
}
}
}
- if (! foundAnyLogs) {
+ if (!foundAnyLogs) {
emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString());
return -1;
}
@@ -398,6 +450,9 @@ public class LogCLIHelpers implements Configurable {
getConf()).listStatus(remoteAppLogDir);
} catch (FileNotFoundException fnf) {
logDirNotExist(remoteAppLogDir.toString());
+ } catch (AccessControlException | AccessDeniedException ace) {
+ logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner,
+ ace.getMessage());
}
return nodeFiles;
}
@@ -426,7 +481,7 @@ public class LogCLIHelpers implements Configurable {
private static void containerLogNotFound(String containerId) {
System.err.println("Logs for container " + containerId
- + " are not present in this log-file.");
+ + " are not present in this log-file.");
}
private static void logDirNotExist(String remoteAppLogDir) {
@@ -437,4 +492,13 @@ public class LogCLIHelpers implements Configurable {
private static void emptyLogDir(String remoteAppLogDir) {
System.err.println(remoteAppLogDir + " does not have any log files.");
}
+
+ private static void logDirNoAccessPermission(String remoteAppLogDir,
+ String appOwner, String errorMessage) throws IOException {
+ System.err.println("Guessed logs' owner is " + appOwner
+ + " and current user "
+ + UserGroupInformation.getCurrentUser().getUserName() + " does not "
+ + "have permission to access " + remoteAppLogDir
+ + ". Error message found: " + errorMessage);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org