You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ar...@apache.org on 2011/10/04 03:12:13 UTC
svn commit: r1178661 - in /incubator/sqoop/trunk/src:
java/com/cloudera/sqoop/manager/SqlManager.java
java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
test/com/cloudera/sqoop/manager/DirectMySQLTest.java
Author: arvind
Date: Tue Oct 4 01:12:13 2011
New Revision: 1178661
URL: http://svn.apache.org/viewvc?rev=1178661&view=rev
Log:
SQOOP-341. Support for unsigned integers with MySQL.
(Alex Newman via Arvind Prabhakar)
Modified:
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java Tue Oct 4 01:12:13 2011
@@ -206,6 +206,12 @@ public abstract class SqlManager extends
ResultSetMetaData metadata = results.getMetaData();
for (int i = 1; i < cols + 1; i++) {
int typeId = metadata.getColumnType(i);
+ // If we have an unsigned int we need to make extra room by
+ // plopping it into a bigint
+ if (typeId == Types.INTEGER && !metadata.isSigned(i)){
+ typeId = Types.BIGINT;
+ }
+
String colName = metadata.getColumnName(i);
if (colName == null || colName.equals("")) {
colName = metadata.getColumnLabel(i);
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java Tue Oct 4 01:12:13 2011
@@ -199,6 +199,13 @@ public class DataDrivenDBInputFormat<T e
// for interpolating split points (i.e., numeric splits, text splits,
// dates, etc.)
int sqlDataType = results.getMetaData().getColumnType(1);
+ boolean isSigned = results.getMetaData().isSigned(1);
+
+ // MySQL has an unsigned integer which we need to allocate space for
+ if (sqlDataType == Types.INTEGER && !isSigned){
+ sqlDataType = Types.BIGINT;
+ }
+
DBSplitter splitter = getSplitter(sqlDataType);
if (null == splitter) {
throw new IOException("Unknown SQL data type: " + sqlDataType);
Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java Tue Oct 4 01:12:13 2011
@@ -105,16 +105,17 @@ public class DirectMySQLTest extends Imp
st.executeUpdate("CREATE TABLE " + getTableName() + " ("
+ "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
+ "name VARCHAR(24) NOT NULL, "
+ + "overly_large_number INT UNSIGNED,"
+ "start_date DATE, "
+ "salary FLOAT, "
+ "dept VARCHAR(32))");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
- + "NULL,'Aaron','2009-05-14',1000000.00,'engineering')");
+ + "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
- + "NULL,'Bob','2009-04-20',400.00,'sales')");
+ + "NULL,'Bob',100,'2009-04-20',400.00,'sales')");
st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
- + "NULL,'Fred','2009-01-23',15.00,'marketing')");
+ + "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')");
connection.commit();
} catch (SQLException sqlE) {
LOG.error("Encountered SQL Exception: " + sqlE);
@@ -231,8 +232,8 @@ public class DirectMySQLTest extends Imp
public void testDirectBulkImportWithDefaultDelims() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
+ "2,Bob,100,2009-04-20,400,sales",
+ "3,Fred,4000000000,2009-01-23,15,marketing",
};
doImport(false, true, getTableName(), expectedResults, null);
@@ -242,8 +243,8 @@ public class DirectMySQLTest extends Imp
public void testWithExtraParams() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
+ "2,Bob,100,2009-04-20,400,sales",
+ "3,Fred,4000000000,2009-01-23,15,marketing",
};
String [] extraArgs = { "--", "--lock-tables" };
@@ -255,8 +256,8 @@ public class DirectMySQLTest extends Imp
public void testMultiMappers() throws IOException {
// no quoting of strings allowed.
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
+ "2,Bob,100,2009-04-20,400,sales",
+ "3,Fred,4000000000,2009-01-23,15,marketing",
};
String [] extraArgs = { "-m", "2" };
@@ -297,8 +298,8 @@ public class DirectMySQLTest extends Imp
public void testDirectBulkImportWithMySQLQuotes() throws IOException {
// mysql quotes all string-based output.
String [] expectedResults = {
- "2,'Bob','2009-04-20',400,'sales'",
- "3,'Fred','2009-01-23',15,'marketing'",
+ "2,'Bob',100,'2009-04-20',400,'sales'",
+ "3,'Fred',4000000000,'2009-01-23',15,'marketing'",
};
doImport(true, true, getTableName(), expectedResults, null);
@@ -307,8 +308,8 @@ public class DirectMySQLTest extends Imp
@Test
public void testMySQLJdbcImport() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400.0,sales",
- "3,Fred,2009-01-23,15.0,marketing",
+ "2,Bob,100,2009-04-20,400.0,sales",
+ "3,Fred,4000000000,2009-01-23,15.0,marketing",
};
doImport(false, false, getTableName(), expectedResults, null);