You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2011/07/28 06:08:36 UTC
svn commit: r1151722 - in /hadoop/common/trunk/mapreduce: CHANGES.txt
src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
Author: szetszwo
Date: Thu Jul 28 04:08:34 2011
New Revision: 1151722
URL: http://svn.apache.org/viewvc?rev=1151722&view=rev
Log:
MAPREDUCE-2463. Job history files are not moved to done folder when job history location is hdfs. Contributed by Devaraj K
Modified:
hadoop/common/trunk/mapreduce/CHANGES.txt
hadoop/common/trunk/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
Modified: hadoop/common/trunk/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/mapreduce/CHANGES.txt?rev=1151722&r1=1151721&r2=1151722&view=diff
==============================================================================
--- hadoop/common/trunk/mapreduce/CHANGES.txt (original)
+++ hadoop/common/trunk/mapreduce/CHANGES.txt Thu Jul 28 04:08:34 2011
@@ -359,6 +359,9 @@ Trunk (unreleased changes)
MAPREDUCE-2732. Remove directly accessing FSNamesystem.LOG from
TestCopyFiles and TestDistCh. (szetszwo)
+ MAPREDUCE-2463. Job history files are not moved to done folder when job
+ history location is hdfs. (Devaraj K via szetszwo)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1151722&r1=1151721&r2=1151722&view=diff
==============================================================================
--- hadoop/common/trunk/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/common/trunk/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Thu Jul 28 04:08:34 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapred.JobConf;
@@ -354,7 +355,8 @@ public class JobHistory {
if (logDirFs.exists(fromPath)) {
LOG.info("Moving " + fromPath.toString() + " to " +
toPath.toString());
- doneDirFs.moveFromLocalFile(fromPath, toPath);
+ FileUtil.copy(logDirFs, fromPath, doneDirFs, toPath, true, false,
+ jobTracker.getConf());
doneDirFs.setPermission(toPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION));
}
Modified: hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java?rev=1151722&r1=1151721&r2=1151722&view=diff
==============================================================================
--- hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java Thu Jul 28 04:08:34 2011
@@ -76,6 +76,11 @@ public class TestJobHistory extends Test
private static String TEST_ROOT_DIR = new File(System.getProperty(
"test.build.data", "/tmp")).toURI().toString().replace(' ', '+');
+
+ private static final String LOG_DIR = System.getProperty("hadoop.log.dir");
+
+ private static final String LOCAL_LOG_DIR_URI = new File(LOG_DIR).toURI()
+ .toString().replace(' ', '+') + "/history";
private static final String DIGITS = "[0-9]+";
@@ -607,15 +612,32 @@ public class TestJobHistory extends Test
assertTrue(jobInfo.getJobQueueName().equals(conf.getQueueName()));
}
+ /**
+ * Tests the case where the log directory is on local disk, the done folder is on HDFS,
+ * and the default FS is local.
+ */
public void testDoneFolderOnHDFS() throws IOException, InterruptedException {
- runDoneFolderTest("history_done");
+ runDoneFolderTest("history_done", LOCAL_LOG_DIR_URI);
}
+ /**
+ * Tests the case where the log directory and done folder is on local disk
+ * and the default FS is local.
+ */
public void testDoneFolderNotOnDefaultFileSystem() throws IOException, InterruptedException {
- runDoneFolderTest("file://" + System.getProperty("test.build.data", "tmp") + "/history_done");
+ runDoneFolderTest(TEST_ROOT_DIR + "/history_done", LOCAL_LOG_DIR_URI);
+ }
+
+ /**
+ * Tests the case where the log directory is on HDFS and done folder is on local disk
+ * and the default FS is local.
+ */
+ public void testHistoryFolderOnHDFS() throws IOException, InterruptedException {
+ String logDir = "hdfs://localhost:%d/history";
+ runDoneFolderTest(TEST_ROOT_DIR + "/done", logDir);
}
- private void runDoneFolderTest(String doneFolder) throws IOException, InterruptedException {
+ private void runDoneFolderTest(String doneFolder, String historyFolder) throws IOException, InterruptedException {
MiniMRCluster mr = null;
MiniDFSCluster dfsCluster = null;
try {
@@ -627,9 +649,11 @@ public class TestJobHistory extends Test
//set the done folder location
conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder);
- String logDir =
- "file:///" + new File(System.getProperty("hadoop.log.dir")).
- getAbsolutePath() + File.separator + "history";
+ dfsCluster = new MiniDFSCluster(conf, 2, true, null);
+ String logDir = String.format(historyFolder, dfsCluster.getNameNodePort());
+
+ //set the history folder location
+ conf.set(JTConfig.JT_JOBHISTORY_LOCATION, logDir);
Path logDirPath = new Path(logDir);
FileSystem logDirFs = logDirPath.getFileSystem(conf);
@@ -647,7 +671,6 @@ public class TestJobHistory extends Test
assertEquals("No of file in logDir not correct", 2,
logDirFs.listStatus(logDirPath).length);
- dfsCluster = new MiniDFSCluster(conf, 2, true, null);
mr = new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(),
3, null, null, conf);