You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2014/12/02 02:39:47 UTC
sqoop git commit: SQOOP-1819: Support Enum in CSVIDF ( + add unit
tests)
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 d68c05d3a -> 38f2894ed
SQOOP-1819: Support Enum in CSVIDF ( + add unit tests)
(Veena Basavaraj via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/38f2894e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/38f2894e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/38f2894e
Branch: refs/heads/sqoop2
Commit: 38f2894ed6119ebad5cbd6e2a523ec25caf64c8d
Parents: d68c05d
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Dec 1 17:38:53 2014 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Mon Dec 1 17:38:53 2014 -0800
----------------------------------------------------------------------
.../idf/CSVIntermediateDataFormat.java | 8 ++++-
.../idf/TestCSVIntermediateDataFormat.java | 34 +++++++++++++-------
2 files changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/38f2894e/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index bdab7a4..bd0fbf0 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -128,7 +128,7 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
List<Column> columns = schema.getColumns();
int i = 0;
for (Column col : columns) {
- if (col.getType() == ColumnType.TEXT) {
+ if (isColumnStringType(col) ) {
stringTypeColumnIndices.add(i);
} else if (col.getType() == ColumnType.BINARY) {
byteTypeColumnIndices.add(i);
@@ -232,6 +232,7 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
Object returnValue = null;
switch (column.getType()) {
+ case ENUM:
case TEXT:
returnValue = unescapeString(fieldString);
break;
@@ -453,6 +454,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
return listType.getType().equals(ColumnType.ARRAY) || listType.getType().equals(ColumnType.SET);
}
+ private boolean isColumnStringType(Column stringType) {
+ return stringType.getType().equals(ColumnType.TEXT)
+ || stringType.getType().equals(ColumnType.ENUM);
+ }
+
private String escapeByteArrays(byte[] bytes) {
try {
return escapeString(new String(bytes, BYTE_FIELD_CHARSET));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/38f2894e/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index bd082aa..75fe429 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -99,7 +99,7 @@ public class TestCSVIntermediateDataFormat {
@Test
public void testInputAsCSVTextInCSVTextOut() {
- String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ String testData = "'ENUM',10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ ",'" + String.valueOf(0x0A) + "'";
dataFormat.setTextData(testData);
assertEquals(testData, dataFormat.getTextData());
@@ -110,14 +110,15 @@ public class TestCSVIntermediateDataFormat {
//byte[0] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
- + ",'\\n'";
+ + ",'\\n','TEST_ENUM'";
Schema schema = new Schema("test");
schema.addColumn(new FixedPoint("1"))
.addColumn(new FixedPoint("2"))
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
- .addColumn(new Text("6"));
+ .addColumn(new Text("6"))
+ .addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
dataFormat.setTextData(testData);
@@ -131,6 +132,7 @@ public class TestCSVIntermediateDataFormat {
assertEquals(-112, ((byte[]) out[4])[0]);
assertEquals(54, ((byte[])out[4])[1]);
assertEquals("\n", out[5].toString());
+ assertEquals("TEST_ENUM", out[6].toString());
}
@Test
@@ -141,23 +143,25 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
- .addColumn(new Text("6"));
+ .addColumn(new Text("6"))
+ .addColumn(new org.apache.sqoop.schema.type.Enum("7"));
dataFormat.setSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
- Object[] in = new Object[6];
+ Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = "54";
in[3] = "random data";
in[4] = byteFieldData;
in[5] = new String(new char[] { 0x0A });
+ in[6] = "TEST_ENUM";
dataFormat.setObjectData(in);
//byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,'54','random data'," +
- getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
+ getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n','TEST_ENUM'";
assertEquals(testData, dataFormat.getTextData());
}
@@ -171,17 +175,20 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
- .addColumn(new Text("6"));
+ .addColumn(new Text("6"))
+ .addColumn(new org.apache.sqoop.schema.type.Enum("7"));
+
dataFormat.setSchema(schema);
- Object[] in = new Object[6];
+ Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = "54";
in[3] = "random data";
in[4] = new byte[] { (byte) -112, (byte) 54};
in[5] = new String(new char[] { 0x0A });
- Object[] inCopy = new Object[6];
+ in[6] = "TEST_ENUM";
+ Object[] inCopy = new Object[7];
System.arraycopy(in,0,inCopy,0,in.length);
// Modifies the input array, so we use the copy to confirm
@@ -198,23 +205,26 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("3"))
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
- .addColumn(new Text("6"));
+ .addColumn(new Text("6"))
+ .addColumn(new org.apache.sqoop.schema.type.Enum("7"));
+
dataFormat.setSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
- Object[] in = new Object[6];
+ Object[] in = new Object[7];
in[0] = new Long(10);
in[1] = new Long(34);
in[2] = null;
in[3] = "random data";
in[4] = byteFieldData;
in[5] = new String(new char[] { 0x0A });
+ in[6] = "TEST_ENUM";
dataFormat.setObjectData(in);
//byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
String testData = "10,34,NULL,'random data'," +
- getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
+ getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n','TEST_ENUM'";
assertEquals(testData, dataFormat.getTextData());
}