You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/11/09 08:31:35 UTC
sqoop git commit: SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat
is failing on Hadoop 1 profile
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 a389c2c7c -> 0e4f46b44
SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat is failing on Hadoop 1 profile
(Jarek Jarcec Cecho via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0e4f46b4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0e4f46b4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0e4f46b4
Branch: refs/heads/sqoop2
Commit: 0e4f46b44d66e9418e3e81614b688f7352f026b3
Parents: a389c2c
Author: Abraham Elmahrek <ab...@elmahrek.com>
Authored: Sat Nov 8 23:26:08 2014 -0800
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Sat Nov 8 23:26:35 2014 -0800
----------------------------------------------------------------------
.../java/org/apache/sqoop/job/JobUtils.java | 41 +++++++++++++++++++-
1 file changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0e4f46b4/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
index 7ce8011..dafdeb4 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
@@ -23,10 +23,16 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.sqoop.job.io.SqoopWritable;
import org.apache.sqoop.job.mr.SqoopSplit;
+import org.apache.sqoop.utils.ClassUtils;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class JobUtils {
@@ -44,7 +50,40 @@ public class JobUtils {
job.setOutputKeyClass(SqoopWritable.class);
job.setOutputValueClass(NullWritable.class);
- return job.waitForCompletion(true);
+ boolean ret = job.waitForCompletion(true);
+
+ // Hadoop 1.0 (and 0.20) have nasty bug when job committer is not called in LocalJobRuner
+ if (isHadoop1()) {
+ callOutputCommitter(job, output);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Call output format on given job manually.
+ */
+ private static void callOutputCommitter(Job job, Class<? extends OutputFormat<SqoopWritable, NullWritable>> outputFormat) throws IOException, InterruptedException {
+ OutputCommitter committer = ((OutputFormat)ClassUtils.instantiate(outputFormat)).getOutputCommitter(null);
+
+ JobContext jobContext = mock(JobContext.class);
+ when(jobContext.getConfiguration()).thenReturn(job.getConfiguration());
+
+ committer.commitJob(jobContext);
+ }
+
+ /**
+ * Detect Hadoop 1.0 installation
+ *
+ * @return True if and only if this is Hadoop 1 and below
+ */
+ public static boolean isHadoop1() {
+ String version = org.apache.hadoop.util.VersionInfo.getVersion();
+ if (version.matches("\\b0\\.20\\..+\\b")
+ || version.matches("\\b1\\.\\d\\.\\d")) {
+ return true;
+ }
+ return false;
}
private JobUtils() {