You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2022/07/11 06:04:59 UTC
[hadoop] branch trunk updated: MAPREDUCE-7201.Make Job History File Permissions configurable (#4507)
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4e8c0b902ec MAPREDUCE-7201.Make Job History File Permissions configurable (#4507)
4e8c0b902ec is described below
commit 4e8c0b902ec199af66dad60c454e16cc2114985a
Author: Ashutosh Gupta <as...@st.niituniversity.in>
AuthorDate: Mon Jul 11 07:04:52 2022 +0100
MAPREDUCE-7201.Make Job History File Permissions configurable (#4507)
* MAPREDUCE-7201.Make Job History File Permissions configurable
Co-authored-by: Ashutosh Gupta <as...@amazon.com>
---
.../jobhistory/JobHistoryEventHandler.java | 7 ++--
.../jobhistory/TestJobHistoryEventHandler.java | 44 ++++++++++++++++++++++
.../mapreduce/v2/jobhistory/JobHistoryUtils.java | 5 +--
3 files changed, 49 insertions(+), 7 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index fc9254af97a..9023f6a11ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1481,7 +1481,7 @@ public class JobHistoryEventHandler extends AbstractService
summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
summaryFileOut.close();
doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
- JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+ JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig())));
} catch (IOException e) {
LOG.info("Unable to write out JobSummaryInfo to ["
+ qualifiedSummaryDoneFile + "]", e);
@@ -1738,8 +1738,9 @@ public class JobHistoryEventHandler extends AbstractService
boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
false, getConfig());
- doneDirFS.setPermission(toPath, new FsPermission(
- JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+ doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils.
+ getConfiguredHistoryIntermediateUserDoneDirPermissions(
+ getConfig())));
if (copied) {
LOG.info("Copied from: " + fromPath.toString()
+ " to done location: " + toPath.toString());
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index fbeb94a2a16..8159bc2456c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.mapreduce.jobhistory;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -43,6 +44,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.CounterGroup;
@@ -1036,6 +1038,48 @@ public class TestJobHistoryEventHandler {
jheh.stop();
}
}
+
+ @Test(timeout = 50000)
+ public void testJobHistoryFilePermissions() throws Exception {
+ TestParams t = new TestParams(true);
+ Configuration conf = new Configuration();
+ String setFilePermission = "777";
+ conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission);
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString());
+
+ JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext,
+ 0, false);
+ JHEvenHandlerForTest jheh = spy(realJheh);
+ jheh.init(conf);
+
+ try {
+ jheh.start();
+ handleEvent(jheh, new JobHistoryEvent(t.jobId,
+ new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost",
+ 3000, 4000, -1)));
+
+ // Job finishes and successfully writes history
+ handleEvent(jheh, new JobHistoryEvent(t.jobId,
+ new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0,
+ 0, 0, 0, 0, 0,
+ new Counters(),
+ new Counters(), new Counters())));
+
+ verify(jheh, times(1)).processDoneFiles(any(JobId.class));
+
+ String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId);
+ String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+ FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0));
+ Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName);
+ FsPermission getIntermediateSummaryFilePermission =
+ fs.getFileStatus(intermediateSummaryFileNamePath).getPermission();
+ assertEquals(setFilePermission,
+ String.valueOf(getIntermediateSummaryFilePermission.toOctal()));
+ } finally {
+ jheh.stop();
+ }
+ }
}
class JHEvenHandlerForTest extends JobHistoryEventHandler {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
index e22726c303a..1a1df010dd8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
@@ -91,9 +91,6 @@ public class JobHistoryUtils {
public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 01777);
- public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS =
- FsPermission.createImmutable((short) 0770); // rwx------
-
/**
* Suffix for configuration files.
*/
@@ -206,7 +203,7 @@ public class JobHistoryUtils {
/**
* Gets the configured directory permissions for the user directories in the
- * directory of the intermediate done history files. The user and the group
+ * Gets the configured permissions for the user directories and files in the
* both need full permissions, this is enforced by this method.
* @param conf The configuration object
* @return FsPermission of the user directories
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org