You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/11/01 20:24:06 UTC
hadoop git commit: YARN-5001. Aggregated Logs root directory is
created with wrong group if nonexistent. Contributed by Haibo Chen
Repository: hadoop
Updated Branches:
refs/heads/trunk 34173a42d -> 76893a410
YARN-5001. Aggregated Logs root directory is created with wrong group if nonexistent. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76893a41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76893a41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76893a41
Branch: refs/heads/trunk
Commit: 76893a41003d57d94eb1a5f486010815266046af
Parents: 34173a4
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 1 20:23:24 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 1 20:23:24 2016 +0000
----------------------------------------------------------------------
.../logaggregation/LogAggregationService.java | 15 ++++++++++++
.../TestLogAggregationService.java | 24 +++++++++++++++++++-
2 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76893a41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index a4ae643..38315fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -258,6 +258,21 @@ public class LogAggregationService extends AbstractService implements
remoteFS.getWorkingDirectory());
remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ String primaryGroupName = null;
+ try {
+ primaryGroupName = loginUser.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.warn("No primary group found. The remote root log directory" +
+ " will be created with the HDFS superuser being its group " +
+ "owner. JobHistoryServer may be unable to read the directory.");
+ }
+ // set owner on the remote directory only if the primary group exists
+ if (primaryGroupName != null) {
+ remoteFS.setOwner(qualified,
+ loginUser.getShortUserName(), primaryGroupName);
+ }
} catch (IOException e) {
throw new YarnRuntimeException("Failed to create remoteLogDir ["
+ this.remoteRootLogDir + "]", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76893a41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index e7330aa..4e9829a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -608,7 +608,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis()));
- this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
aNewFile.getAbsolutePath());
LogAggregationService logAggregationService = spy(
@@ -626,6 +626,28 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
}
@Test
+ public void testRemoteRootLogDirIsCreatedWithCorrectGroupOwner()
+ throws IOException {
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ Path aNewFile = new Path(String.valueOf("tmp"+System.currentTimeMillis()));
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, aNewFile.getName());
+
+ LogAggregationService logAggregationService = new LogAggregationService(
+ dispatcher, this.context, this.delSrvc, super.dirsHandler);
+ logAggregationService.init(this.conf);
+ logAggregationService.verifyAndCreateRemoteLogDir(this.conf);
+
+ String targetGroup =
+ UserGroupInformation.getLoginUser().getPrimaryGroupName();
+ FileSystem fs = FileSystem.get(this.conf);
+ FileStatus fileStatus = fs.getFileStatus(aNewFile);
+ Assert.assertEquals("The new aggregate file is not successfully created",
+ fileStatus.getGroup(), targetGroup);
+
+ fs.delete(aNewFile, true);
+ }
+
+ @Test
public void testAppLogDirCreation() throws Exception {
final String logSuffix = "logs";
this.conf.set(YarnConfiguration.NM_LOG_DIRS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org