[hadoop] branch trunk updated: YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 01d7924  YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.
01d7924 is described below

commit 01d79244732c7f60dff3cd7181647c0460955491
Author: Sunil G <su...@apache.org>
AuthorDate: Wed Sep 18 09:27:41 2019 +0530

    YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.
---
 .../apache/hadoop/yarn/conf/YarnConfiguration.java |  6 ++
 .../LogAggregationFileController.java              | 25 ++++--
 .../src/main/resources/yarn-default.xml            |  8 ++
 .../TestLogAggregationFileController.java          | 91 ++++++++++++++++++++++
 4 files changed, 122 insertions(+), 8 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9c62827..7b05905 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1479,6 +1479,12 @@ public class YarnConfiguration extends Configuration {
   public static final boolean DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER =
       true;
 
+  /**
+   * Specifies the group of the aggregated log directory.
+   */
+  public static final String NM_REMOTE_APP_LOG_DIR_GROUPNAME =
+      NM_PREFIX + "remote-app-log-dir.groupname";
+
   public static final String YARN_LOG_SERVER_URL =
     YARN_PREFIX + "log.server.url";
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 661e321..001f4f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
 import org.apache.hadoop.yarn.webapp.View.ViewContext;
@@ -346,13 +345,23 @@ public abstract class LogAggregationFileController {
         }
 
         UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
-        String primaryGroupName = null;
-        try {
-          primaryGroupName = loginUser.getPrimaryGroupName();
-        } catch (IOException e) {
-          LOG.warn("No primary group found. The remote root log directory" +
-              " will be created with the HDFS superuser being its group " +
-              "owner. JobHistoryServer may be unable to read the directory.");
+        String primaryGroupName = conf.get(
+            YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME);
+        if (primaryGroupName == null || primaryGroupName.isEmpty()) {
+          try {
+            primaryGroupName = loginUser.getPrimaryGroupName();
+          } catch (IOException e) {
+            LOG.warn("No primary group found. The remote root log directory" +
+                    " will be created with the HDFS superuser being its " +
+                    "group owner. JobHistoryServer may be unable to read " +
+                    "the directory.");
+          }
+        } else {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("The group of remote root log directory has been " +
+                "determined by the configuration and set to " +
+                primaryGroupName);
+          }
         }
         // set owner on the remote directory only if the primary group exists
         if (primaryGroupName != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b856536..55e908d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1374,6 +1374,14 @@
   </property>
 
   <property>
+    <description>If the NodeManager creates the remote-app-log-dir folder,
+    it will be created with this groupname.
+    </description>
+    <name>yarn.nodemanager.remote-app-log-dir.groupname</name>
+    <value></value>
+  </property>
+
+  <property>
     <description>Generate additional logs about container launches.
     Currently, this creates a copy of the launch script and lists the
     directory contents of the container work dir. When listing directory
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
new file mode 100644
index 0000000..5ade0fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.filecontroller;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.FileNotFoundException;
+import java.net.URI;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Test for the abstract {@link LogAggregationFileController} class,
+ * checking its core functionality.
+ */
+public class TestLogAggregationFileController {
+
+  @Test
+  public void testRemoteDirCreationDefault() throws Exception {
+    FileSystem fs = mock(FileSystem.class);
+    doReturn(new URI("")).when(fs).getUri();
+    doThrow(FileNotFoundException.class).when(fs)
+            .getFileStatus(any(Path.class));
+
+    Configuration conf = new Configuration();
+    LogAggregationFileController controller = mock(
+            LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
+    doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
+
+    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+        "yarn_user", new String[] {"yarn_group", "other_group"});
+    UserGroupInformation.setLoginUser(ugi);
+
+    controller.initialize(conf, "TFile");
+    controller.verifyAndCreateRemoteLogDir();
+
+    verify(fs).setOwner(any(), eq("yarn_user"), eq("yarn_group"));
+  }
+
+  @Test
+  public void testRemoteDirCreationWithCustomGroup() throws Exception {
+    String testGroupName = "testGroup";
+
+    FileSystem fs = mock(FileSystem.class);
+    doReturn(new URI("")).when(fs).getUri();
+    doThrow(FileNotFoundException.class).when(fs)
+        .getFileStatus(any(Path.class));
+
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME, testGroupName);
+    LogAggregationFileController controller = mock(
+        LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
+    doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
+
+    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+        "yarn_user", new String[] {"yarn_group", "other_group"});
+    UserGroupInformation.setLoginUser(ugi);
+
+    controller.initialize(conf, "TFile");
+    controller.verifyAndCreateRemoteLogDir();
+
+    verify(fs).setOwner(any(), eq("yarn_user"), eq(testGroupName));
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org