You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/08/27 19:37:13 UTC
[13/29] git commit: MAPREDUCE-6044. Fully qualified intermediate done
dir path breaks per-user dir creation on Windows. Contributed by Zhijie Shen.
MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir creation on Windows. Contributed by Zhijie Shen.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1619863 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/524a63e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/524a63e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/524a63e5
Branch: refs/heads/HDFS-6584
Commit: 524a63e59ecfda99c66842462e1bac5d610e7997
Parents: 0097b15
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri Aug 22 17:06:23 2014 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri Aug 22 17:06:23 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../jobhistory/TestJobHistoryEventHandler.java | 30 +++++++++++++++++++-
.../v2/jobhistory/JobHistoryUtils.java | 4 +--
3 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index dfef8e5..ddf21ed 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -255,6 +255,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6012. DBInputSplit creates invalid ranges on Oracle.
(Wei Yan via kasha)
+ MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir
+ creation on Windows. (zjshen)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index d8a0cc7..7539e73 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -25,7 +25,6 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.never;
import java.io.File;
import java.io.FileOutputStream;
@@ -53,6 +52,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -399,6 +400,33 @@ public class TestJobHistoryEventHandler {
}
}
+ @Test
+ public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
+ // Test relative path
+ Configuration conf = new Configuration();
+ conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
+ "/mapred/history/done_intermediate");
+ conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
+ String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+ Assert.assertEquals("/mapred/history/done_intermediate/" +
+ System.getProperty("user.name"), pathStr);
+
+ // Test fully qualified path
+ // Create default configuration pointing to the minicluster
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ dfsCluster.getURI().toString());
+ FileOutputStream os = new FileOutputStream(coreSitePath);
+ conf.writeXml(os);
+ os.close();
+ // Simulate execution under a non-default namenode
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ "file:///");
+ pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+ Assert.assertEquals(dfsCluster.getURI().toString() +
+ "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
+ pathStr);
+ }
+
private void queueEvent(JHEvenHandlerForTest jheh, JobHistoryEvent event) {
jheh.handle(event);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/524a63e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
index 167ee20..e279c03 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
@@ -292,8 +292,8 @@ public class JobHistoryUtils {
* @return the intermediate done directory for jobhistory files.
*/
public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException {
- return getConfiguredHistoryIntermediateDoneDirPrefix(conf) + File.separator
- + UserGroupInformation.getCurrentUser().getShortUserName();
+ return new Path(getConfiguredHistoryIntermediateDoneDirPrefix(conf),
+ UserGroupInformation.getCurrentUser().getShortUserName()).toString();
}
public static boolean shouldCreateNonUserDirectory(Configuration conf) {