You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2016/02/11 02:04:03 UTC
sqoop git commit: SQOOP-2828: Sqoop2: AvroIntermediateDataFormat
should read Decimals as BigDecimals instead of Strings
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 e64598af4 -> edb42dbdc
SQOOP-2828: Sqoop2: AvroIntermediateDataFormat should read Decimals as BigDecimals instead of Strings
(Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/edb42dbd
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/edb42dbd
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/edb42dbd
Branch: refs/heads/sqoop2
Commit: edb42dbdc29f3834a7bb4eea291e15c8fce77053
Parents: e64598a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Feb 10 17:03:39 2016 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Feb 10 17:03:39 2016 -0800
----------------------------------------------------------------------
.../idf/AVROIntermediateDataFormat.java | 6 ++++--
.../idf/TestAVROIntermediateDataFormat.java | 22 +++++++++++---------
2 files changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/edb42dbd/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
index e409fc1..b55f7a0 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
@@ -352,7 +352,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe
break;
case DECIMAL:
// stored as string
- csvString.append(toCSVDecimal(obj));
+ csvString.append(toCSVDecimal(new BigDecimal(obj.toString())));
break;
case DATE:
// stored as long
@@ -417,9 +417,11 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe
// stored as enum symbol
case TEXT:
// stored as UTF8
+ object[nameIndex] = obj.toString();
+ break;
case DECIMAL:
// stored as string
- object[nameIndex] = obj.toString();
+ object[nameIndex] = new BigDecimal(obj.toString());
break;
case BINARY:
case UNKNOWN:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/edb42dbd/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java
index 8475720..3c4d7de 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java
@@ -35,12 +35,14 @@ import org.apache.sqoop.schema.type.Array;
import org.apache.sqoop.schema.type.Binary;
import org.apache.sqoop.schema.type.Bit;
import org.apache.sqoop.schema.type.Column;
+import org.apache.sqoop.schema.type.Decimal;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.Text;
import org.joda.time.LocalDateTime;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -80,7 +82,7 @@ public class TestAVROIntermediateDataFormat {
enumCol = new org.apache.sqoop.schema.type.Enum("seven").setOptions(options);
sqoopSchema
.addColumn(new FixedPoint("one", 8L, true))
- .addColumn(new FixedPoint("two", 2L, true))
+ .addColumn(new Decimal("two", 4, 2))
.addColumn(new Text("three"))
.addColumn(new Text("four"))
.addColumn(new Binary("five"))
@@ -105,7 +107,7 @@ public class TestAVROIntermediateDataFormat {
@Test
public void testInputAsCSVTextInAndDataOut() {
- String csvText = "10,34,'54','random data',"
+ String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@@ -116,7 +118,7 @@ public class TestAVROIntermediateDataFormat {
@Test
public void testInputAsCSVTextInAndObjectArrayOut() {
- String csvText = "10,34,'54','random data',"
+ String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@@ -129,7 +131,7 @@ public class TestAVROIntermediateDataFormat {
private void assertObjectArray() {
Object[] out = dataFormat.getObjectData();
assertEquals(10L, out[0]);
- assertEquals(34, out[1]);
+ assertEquals(new BigDecimal("34.56"), out[1]);
assertEquals("54", out[2]);
assertEquals("random data", out[3]);
assertEquals(-112, ((byte[]) out[4])[0]);
@@ -173,7 +175,7 @@ public class TestAVROIntermediateDataFormat {
@Test
public void testInputAsCSVTextInCSVTextOut() {
- String csvText = "10,34,'54','random data',"
+ String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@@ -184,7 +186,7 @@ public class TestAVROIntermediateDataFormat {
private GenericRecord createAvroGenericRecord() {
GenericRecord avroObject = new GenericData.Record(avroSchema);
avroObject.put("one", 10L);
- avroObject.put("two", 34);
+ avroObject.put("two", "34.56");
avroObject.put("three", new Utf8("54"));
avroObject.put("four", new Utf8("random data"));
// store byte array in byte buffer
@@ -235,7 +237,7 @@ public class TestAVROIntermediateDataFormat {
@Test
public void testInputAsDataInAndCSVOut() {
- String csvExpected = "10,34,'54','random data',"
+ String csvExpected = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@@ -260,7 +262,7 @@ public class TestAVROIntermediateDataFormat {
private Object[] createObjectArray() {
Object[] out = new Object[15];
out[0] = 10L;
- out[1] = 34;
+ out[1] = new BigDecimal("34.56");
out[2] = "54";
out[3] = "random data";
out[4] = new byte[] { (byte) -112, (byte) 54 };
@@ -322,7 +324,7 @@ public class TestAVROIntermediateDataFormat {
public void testInputAsObjectArrayInAndCSVOut() {
Object[] out = createObjectArray();
dataFormat.setObjectData(out);
- String csvText = "10,34,'54','random data',"
+ String csvText = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A)
+ "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + ","
+ csvDate + ",13.44," + csvSet;
@@ -339,7 +341,7 @@ public class TestAVROIntermediateDataFormat {
// **************test cases for empty and null schema*******************
@Test(expectedExceptions = SqoopException.class)
public void testEmptySchema() {
- String testData = "10,34,'54','random data',"
+ String testData = "10,34.56,'54','random data',"
+ getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'\\n'";
// no coumns
Schema schema = new Schema("Test");