You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/12/24 00:09:08 UTC
sqoop git commit: SQOOP-1935: Sqoop2: Fix TestSqoopWritable test and
make getString and setString package private
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 f81632d8f -> 13c01625a
SQOOP-1935: Sqoop2: Fix TestSqoopWritable test and make getString and setString package private
(Veena Basavaraj via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/13c01625
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/13c01625
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/13c01625
Branch: refs/heads/sqoop2
Commit: 13c01625ad416f44c2e27a2ecb3d6abcca4f116a
Parents: f81632d
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Tue Dec 23 15:07:53 2014 -0800
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Tue Dec 23 15:07:53 2014 -0800
----------------------------------------------------------------------
.../org/apache/sqoop/job/io/SqoopWritable.java | 15 ++---
.../job/mr/SqoopOutputFormatLoadExecutor.java | 2 +-
.../apache/sqoop/job/io/TestSqoopWritable.java | 62 ++++++++++++--------
.../apache/sqoop/job/util/MRJobTestUtil.java | 14 ++---
4 files changed, 55 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13c01625/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
index 6a0bfa4..5967f5c 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
@@ -31,12 +31,16 @@ import java.io.IOException;
/**
* Writable used to load the data to the {@link #Transferable} entity TO
+ * It is used for the map output key class and the outputFormat key class in the MR engine
+ * For instance: job.setMapOutputKeyClass(..,) and job.setOutputKeyClass(...);
*/
public class SqoopWritable implements Configurable, WritableComparable<SqoopWritable> {
private IntermediateDataFormat<?> toIDF;
private Configuration conf;
+ // NOTE: You have to provide an empty default constructor in your key class
+ // Hadoop is using reflection and it can not guess any parameters to feed
public SqoopWritable() {
this(null);
}
@@ -45,14 +49,11 @@ public class SqoopWritable implements Configurable, WritableComparable<SqoopWrit
this.toIDF = dataFormat;
}
- public void setString(String data) {
+ // default/package visibility for testing
+ void setString(String data) {
this.toIDF.setCSVTextData(data);
}
- public String getString() {
- return toIDF.getCSVTextData();
- }
-
@Override
public void write(DataOutput out) throws IOException {
//delegate
@@ -67,12 +68,12 @@ public class SqoopWritable implements Configurable, WritableComparable<SqoopWrit
@Override
public int compareTo(SqoopWritable o) {
- return getString().compareTo(o.getString());
+ return toString().compareTo(o.toString());
}
@Override
public String toString() {
- return getString();
+ return toIDF.getCSVTextData();
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13c01625/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
index 7835e38..58a9eed 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
@@ -102,7 +102,7 @@ public class SqoopOutputFormatLoadExecutor {
free.acquire();
checkIfConsumerThrew();
// NOTE: this is the place where data written from SqoopMapper writable is available to the SqoopOutputFormat
- toDataFormat.setCSVTextData(key.getString());
+ toDataFormat.setCSVTextData(key.toString());
filled.release();
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13c01625/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
index b07a076..79d6a8f 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
@@ -18,6 +18,11 @@
*/
package org.apache.sqoop.job.io;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -28,61 +33,72 @@ import java.io.IOException;
import java.io.InputStream;
import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
-import org.junit.Assert;
+import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.junit.Before;
import org.junit.Test;
public class TestSqoopWritable {
- private final SqoopWritable writable = new SqoopWritable(new CSVIntermediateDataFormat());
+ private SqoopWritable writable;
+ private IntermediateDataFormat<?> idfMock;
+
+ @Before
+ public void setUp() {
+ idfMock = mock(IntermediateDataFormat.class);
+ writable = new SqoopWritable(idfMock);
+ }
@Test
public void testStringInStringOut() {
String testData = "Live Long and prosper";
writable.setString(testData);
- Assert.assertEquals(testData,writable.getString());
+ verify(idfMock, times(1)).setCSVTextData(testData);
+ writable.toString();
+ verify(idfMock, times(1)).getCSVTextData();
}
@Test
- public void testDataWritten() throws IOException {
+ public void testWrite() throws IOException {
String testData = "One ring to rule them all";
- writable.setString(testData);
ByteArrayOutputStream ostream = new ByteArrayOutputStream();
+ ostream.write(testData.getBytes());
DataOutput out = new DataOutputStream(ostream);
writable.write(out);
- byte[] written = ostream.toByteArray();
- InputStream instream = new ByteArrayInputStream(written);
- DataInput in = new DataInputStream(instream);
- String readData = in.readUTF();
- Assert.assertEquals(testData, readData);
+ // verify that the idf method is called, that is all the test should test
+ verify(idfMock, times(1)).write(out);
+ ostream.close();
}
@Test
- public void testDataRead() throws IOException {
+ public void testReadFields() throws IOException {
String testData = "Brandywine Bridge - 20 miles!";
- ByteArrayOutputStream ostream = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(ostream);
- out.writeUTF(testData);
- InputStream instream = new ByteArrayInputStream(ostream.toByteArray());
+ InputStream instream = new ByteArrayInputStream(testData.getBytes());
DataInput in = new DataInputStream(instream);
writable.readFields(in);
- Assert.assertEquals(testData, writable.getString());
+ // verify that the idf method is called, that is all the test should test
+ verify(idfMock, times(1)).read(in);
+ instream.close();
}
+ // NOTE: This test is testing that the write and readFields methods work
+ // and not really testing anything about SqoopWritable. Have kept this test since
+ // it existed before.
@Test
- public void testWriteReadUsingStream() throws IOException {
+ public void testWriteAndReadFields() throws IOException {
String testData = "You shall not pass";
ByteArrayOutputStream ostream = new ByteArrayOutputStream();
DataOutput out = new DataOutputStream(ostream);
- writable.setString(testData);
- writable.write(out);
+ SqoopWritable writableOne = new SqoopWritable(new CSVIntermediateDataFormat());
+ writableOne.setString(testData);
+ writableOne.write(out);
byte[] written = ostream.toByteArray();
- //Don't test what the data is, test that SqoopWritable can read it.
+ // Don't test what the data is, test that SqoopWritable can read it.
InputStream instream = new ByteArrayInputStream(written);
- SqoopWritable newWritable = new SqoopWritable(new CSVIntermediateDataFormat());
+ SqoopWritable writableTwo = new SqoopWritable(new CSVIntermediateDataFormat());
DataInput in = new DataInputStream(instream);
- newWritable.readFields(in);
- Assert.assertEquals(testData, newWritable.getString());
+ writableTwo.readFields(in);
+ assertEquals(writableOne.toString(), writableTwo.toString());
ostream.close();
instream.close();
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13c01625/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
index 5d5359e..d498850 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
@@ -44,16 +44,16 @@ public class MRJobTestUtil {
@SuppressWarnings("deprecation")
public static boolean runJob(Configuration conf,
- Class<? extends InputFormat<SqoopSplit, NullWritable>> input,
- Class<? extends Mapper<SqoopSplit, NullWritable, SqoopWritable, NullWritable>> mapper,
- Class<? extends OutputFormat<SqoopWritable, NullWritable>> output) throws IOException,
+ Class<? extends InputFormat<SqoopSplit, NullWritable>> inputFormatClass,
+ Class<? extends Mapper<SqoopSplit, NullWritable, SqoopWritable, NullWritable>> mapperClass,
+ Class<? extends OutputFormat<SqoopWritable, NullWritable>> outputFormatClass) throws IOException,
InterruptedException, ClassNotFoundException {
Job job = new Job(conf);
- job.setInputFormatClass(input);
- job.setMapperClass(mapper);
+ job.setInputFormatClass(inputFormatClass);
+ job.setMapperClass(mapperClass);
job.setMapOutputKeyClass(SqoopWritable.class);
job.setMapOutputValueClass(NullWritable.class);
- job.setOutputFormatClass(output);
+ job.setOutputFormatClass(outputFormatClass);
job.setOutputKeyClass(SqoopWritable.class);
job.setOutputValueClass(NullWritable.class);
@@ -62,7 +62,7 @@ public class MRJobTestUtil {
// Hadoop 1.0 (and 0.20) have nasty bug when job committer is not called in
// LocalJobRuner
if (isHadoop1()) {
- callOutputCommitter(job, output);
+ callOutputCommitter(job, outputFormatClass);
}
return ret;