svn commit: r713423 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java

[jo...@apache.org]

Author: johan
Date: Wed Nov 12 08:48:08 2008
New Revision: 713423

URL: http://svn.apache.org/viewvc?rev=713423&view=rev
Log:
HADOOP-4505. Add a unit test to test faulty setup task and cleanup task killing the job. (Amareshwari Sriramadasu via johan)

Added:
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java
Modified:
    hadoop/core/trunk/CHANGES.txt

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=713423&r1=713422&r2=713423&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Nov 12 08:48:08 2008
@@ -91,7 +91,10 @@
     HADOOP-3923. Remove org.apache.hadoop.mapred.StatusHttpServer.  (szetszwo)
     
     HADOOP-4622. Explicitly specify interpretor for non-native
-    pipes binaries. (Fredrik Hedberg via johan)    
+    pipes binaries. (Fredrik Hedberg via johan)
+    
+    HADOOP-4505. Add a unit test to test faulty setup task and cleanup
+    task killing the job. (Amareshwari Sriramadasu via johan)
 
   OPTIMIZATIONS
 

Added: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java?rev=713423&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java Wed Nov 12 08:48:08 2008
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.lib.IdentityMapper;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class TestSetupAndCleanupFailure extends TestCase {
+
+  static class CommitterWithFailSetup extends FileOutputCommitter {
+    public void setupJob(JobContext context) throws IOException {
+      throw new IOException();
+    }
+  }
+
+  static class CommitterWithFailCleanup extends FileOutputCommitter {
+    public void cleanupJob(JobContext context) throws IOException {
+      throw new IOException();
+    }
+  }
+  
+  public RunningJob launchJob(JobConf conf,
+                              Path inDir,
+                              Path outDir,
+                              String input) 
+  throws IOException {
+    // set up the input file system and write input text.
+    FileSystem inFs = inDir.getFileSystem(conf);
+    FileSystem outFs = outDir.getFileSystem(conf);
+    outFs.delete(outDir, true);
+    if (!inFs.mkdirs(inDir)) {
+      throw new IOException("Mkdirs failed to create " + inDir.toString());
+    }
+    {
+      // write input into input file
+      DataOutputStream file = inFs.create(new Path(inDir, "part-0"));
+      file.writeBytes(input);
+      file.close();
+    }
+
+    // configure the mapred Job
+    conf.setMapperClass(IdentityMapper.class);        
+    conf.setReducerClass(IdentityReducer.class);
+    FileInputFormat.setInputPaths(conf, inDir);
+    FileOutputFormat.setOutputPath(conf, outDir);
+    String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data",
+                                    "/tmp")).toString().replace(' ', '+');
+    conf.set("test.build.data", TEST_ROOT_DIR);
+
+    // return the RunningJob handle.
+    return new JobClient(conf).submitJob(conf);
+  }
+  
+  public void testWithDFS() throws IOException {
+    MiniDFSCluster dfs = null;
+    MiniMRCluster mr = null;
+    FileSystem fileSys = null;
+    try {
+      final int taskTrackers = 4;
+
+      Configuration conf = new Configuration();
+      dfs = new MiniDFSCluster(conf, 4, true, null);
+      fileSys = dfs.getFileSystem();
+      mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1);
+      JobConf jobConf = mr.createJobConf();
+      final Path inDir = new Path("./input");
+      final Path outDir = new Path("./output");
+      String input = "The quick brown fox\nhas many silly\nred fox sox\n";
+      RunningJob job = null;
+
+      jobConf.setOutputCommitter(CommitterWithFailSetup.class);
+      job = launchJob(jobConf, inDir, outDir, input);
+      // wait for the job to finish.
+      while (!job.isComplete()) ;
+      assertEquals(JobStatus.FAILED, job.getJobState());
+
+      jobConf.setOutputCommitter(CommitterWithFailCleanup.class);
+      job = launchJob(jobConf, inDir, outDir, input);
+      // wait for the job to finish.
+      while (!job.isComplete()) ;
+      assertEquals(JobStatus.FAILED, job.getJobState());
+    } finally {
+      if (dfs != null) { dfs.shutdown(); }
+      if (mr != null) { mr.shutdown();
+      }
+    }
+  }
+  public static void main(String[] argv) throws Exception {
+    TestSetupAndCleanupFailure td = new TestSetupAndCleanupFailure();
+    td.testWithDFS();
+  }
+}