You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ka...@apache.org on 2013/06/11 00:53:47 UTC
git commit: SQOOP-1071: Avro import is failing when columns are
starting with underscore (Jarek Jarcec Cecho via Kate Ting)
Updated Branches:
refs/heads/trunk 658c15848 -> 5d767e796
SQOOP-1071: Avro import is failing when columns are starting with underscore
(Jarek Jarcec Cecho via Kate Ting)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5d767e79
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5d767e79
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5d767e79
Branch: refs/heads/trunk
Commit: 5d767e796d70830657496558514c349074fbb3eb
Parents: 658c158
Author: Kate Ting <ka...@apache.org>
Authored: Mon Jun 10 17:17:28 2013 -0400
Committer: Kate Ting <ka...@apache.org>
Committed: Mon Jun 10 17:17:28 2013 -0400
----------------------------------------------------------------------
.../apache/sqoop/orm/AvroSchemaGenerator.java | 2 +-
src/test/com/cloudera/sqoop/TestAvroImport.java | 21 +++++++++++++
.../sqoop/testutil/BaseSqoopTestCase.java | 33 +++++++++++++++-----
3 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
index 864fa0d..806bace 100644
--- a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
+++ b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
@@ -60,7 +60,7 @@ public class AvroSchemaGenerator {
List<Field> fields = new ArrayList<Field>();
for (String columnName : columnNames) {
- String cleanedCol = ClassWriter.toIdentifier(columnName);
+ String cleanedCol = ClassWriter.toJavaIdentifier(columnName);
int sqlType = columnTypes.get(columnName);
Schema avroSchema = toAvroSchema(sqlType, columnName);
Field field = new Field(cleanedCol, avroSchema, null, null);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/TestAvroImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroImport.java b/src/test/com/cloudera/sqoop/TestAvroImport.java
index 32d1bbe..34a7d41 100644
--- a/src/test/com/cloudera/sqoop/TestAvroImport.java
+++ b/src/test/com/cloudera/sqoop/TestAvroImport.java
@@ -181,6 +181,27 @@ public class TestAvroImport extends ImportJobTestCase {
assertEquals("DATA_COL0", new Utf8("10"), record1.get("DATA_COL0"));
}
+ public void testFirstUnderscoreInColumnName() throws IOException {
+ String [] names = { "_NAME" };
+ String [] types = { "INT" };
+ String [] vals = { "1987" };
+ createTableWithColTypesAndNames(names, types, vals);
+
+ runImport(getOutputArgv(true, null));
+
+ Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
+ DataFileReader<GenericRecord> reader = read(outputFile);
+ Schema schema = reader.getSchema();
+ assertEquals(Schema.Type.RECORD, schema.getType());
+ List<Field> fields = schema.getFields();
+ assertEquals(types.length, fields.size());
+
+ checkField(fields.get(0), "__NAME", Type.INT);
+
+ GenericRecord record1 = reader.next();
+ assertEquals("__NAME", 1987, record1.get("__NAME"));
+ }
+
private void checkField(Field field, String name, Type type) {
assertEquals(name, field.name());
assertEquals(Schema.Type.UNION, field.schema().getType());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
index d6afbc8..877d7f8 100644
--- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
+++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
@@ -290,18 +290,24 @@ public abstract class BaseSqoopTestCase extends TestCase {
}
/**
- * Create a table with a set of columns and add a row of values.
+ * Create a table with a set of columns with their names and add a row of values.
+ * @param colNames Column names
* @param colTypes the types of the columns to make
* @param vals the SQL text for each value to insert
*/
- protected void createTableWithColTypes(String [] colTypes, String [] vals) {
+ protected void createTableWithColTypesAndNames(String[] colNames,
+ String[] colTypes,
+ String[] vals) {
+ assert colNames != null;
+ assert colTypes != null;
+ assert colNames.length == colTypes.length;
+
Connection conn = null;
PreparedStatement statement = null;
String createTableStr = null;
String columnDefStr = "";
String columnListStr = "";
String valueListStr = "";
- String [] myColNames = new String[colTypes.length];
try {
try {
@@ -310,9 +316,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
conn = getManager().getConnection();
for (int i = 0; i < colTypes.length; i++) {
- String colName = BASE_COL_NAME + Integer.toString(i);
- columnDefStr += colName + " " + colTypes[i];
- myColNames[i] = colName;
+ columnDefStr += colNames[i] + " " + colTypes[i];
if (i < colTypes.length - 1) {
columnDefStr += ", ";
}
@@ -342,7 +346,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
if (vals!=null) {
for (int i = 0; i < colTypes.length; i++) {
- columnListStr += myColNames[i];
+ columnListStr += colNames[i];
valueListStr += vals[i];
if (i < colTypes.length - 1) {
columnListStr += ", ";
@@ -375,7 +379,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
}
conn.commit();
- this.colNames = myColNames;
+ this.colNames = colNames;
} catch (SQLException se) {
if (null != conn) {
try {
@@ -389,6 +393,19 @@ public abstract class BaseSqoopTestCase extends TestCase {
}
/**
+ * Create a table with a set of columns and add a row of values.
+ * @param colTypes the types of the columns to make
+ * @param vals the SQL text for each value to insert
+ */
+ protected void createTableWithColTypes(String [] colTypes, String [] vals) {
+ String[] colNames = new String[colTypes.length];
+ for( int i = 0; i < colTypes.length; i++) {
+ colNames[i] = BASE_COL_NAME + Integer.toString(i);
+ }
+ createTableWithColTypesAndNames(colNames, colTypes, vals);
+ }
+
+ /**
* Create a table with a single column and put a data element in it.
* @param colType the type of the column to create
* @param val the value to insert (reformatted as a string)