svn commit: r1076804 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java

[to...@apache.org]

Author: tomwhite
Date: Thu Mar  3 21:29:31 2011
New Revision: 1076804

URL: http://svn.apache.org/viewvc?rev=1076804&view=rev
Log:
MAPREDUCE-2351. mapred.job.tracker.history.completed.location should support an arbitrary filesystem URI.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1076804&r1=1076803&r2=1076804&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Mar  3 21:29:31 2011
@@ -51,6 +51,9 @@ Trunk (unreleased changes)
 
     MAPREDUCE-2302. Add static factory methods in GaloisField. (schen)
 
+    MAPREDUCE-2351. mapred.job.tracker.history.completed.location should
+    support an arbitrary filesystem URI. (tomwhite)
+
   OPTIMIZATIONS
     
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1076804&r1=1076803&r2=1076804&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Thu Mar  3 21:29:31 2011
@@ -152,8 +152,9 @@ public class JobHistory {
     String doneLocation =
       conf.get(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION);
     if (doneLocation != null) {
-      done = fs.makeQualified(new Path(doneLocation));
-      doneDirFs = fs;
+      Path donePath = new Path(doneLocation);
+      doneDirFs = donePath.getFileSystem(conf);
+      done = doneDirFs.makeQualified(donePath);
     } else {
       done = logDirFs.makeQualified(new Path(logDir, "done"));
       doneDirFs = logDirFs;

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java?rev=1076804&r1=1076803&r2=1076804&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java Thu Mar  3 21:29:31 2011
@@ -606,9 +606,18 @@ public class TestJobHistory extends Test
     // Validate the job queue name
     assertTrue(jobInfo.getJobQueueName().equals(conf.getQueueName()));
   }
-
+  
   public void testDoneFolderOnHDFS() throws IOException, InterruptedException {
+    runDoneFolderTest("history_done");
+  }
+    
+  public void testDoneFolderNotOnDefaultFileSystem() throws IOException, InterruptedException {
+    runDoneFolderTest("file://" + System.getProperty("test.build.data", "tmp") + "/history_done");
+  }
+    
+  private void runDoneFolderTest(String doneFolder) throws IOException, InterruptedException {
     MiniMRCluster mr = null;
+    MiniDFSCluster dfsCluster = null;
     try {
       JobConf conf = new JobConf();
       // keep for less time
@@ -616,7 +625,6 @@ public class TestJobHistory extends Test
       conf.setLong("mapred.jobtracker.retirejob.interval", 1000);
 
       //set the done folder location
-      String doneFolder = "history_done";
       conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder);
 
       String logDir =
@@ -639,7 +647,7 @@ public class TestJobHistory extends Test
       assertEquals("No of file in logDir not correct", 2,
           logDirFs.listStatus(logDirPath).length);
       
-      MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, 2, true, null);
+      dfsCluster = new MiniDFSCluster(conf, 2, true, null);
       mr = new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(),
           3, null, null, conf);
 
@@ -674,7 +682,7 @@ public class TestJobHistory extends Test
       RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
       
       assertEquals("History DONE folder not correct", 
-          doneFolder, doneDir.getName());
+          new Path(doneFolder).getName(), doneDir.getName());
       JobID id = job.getID();
       String logFileName = getDoneFile(jobHistory, conf, id, doneDir);
 
@@ -724,6 +732,9 @@ public class TestJobHistory extends Test
         cleanupLocalFiles(mr);
         mr.shutdown();
       }
+      if (dfsCluster != null) {
+        dfsCluster.shutdown();
+      }
     }
   }