You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/01 17:57:00 UTC
sqoop git commit: SQOOP-2229: Sqoop2: Kite connector should handle
doubles
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 947f53281 -> 213caf071
SQOOP-2229: Sqoop2: Kite connector should handle doubles
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/213caf07
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/213caf07
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/213caf07
Branch: refs/heads/sqoop2
Commit: 213caf07146e53b4c4a841758800990b6fa535e3
Parents: 947f532
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Apr 1 08:56:37 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Apr 1 08:56:37 2015 -0700
----------------------------------------------------------------------
.../sqoop/connector/kite/util/KiteDataTypeUtil.java | 10 +++++++++-
.../connector/hive/FromRDBMSToKiteHiveTest.java | 7 +------
.../connector/kite/FromRDBMSToKiteTest.java | 14 +++++---------
3 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java
index 1074d37..86aa44f 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java
@@ -25,6 +25,7 @@ import org.apache.sqoop.connector.common.FileFormat;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.schema.type.ColumnType;
import org.apache.sqoop.schema.type.FixedPoint;
+import org.apache.sqoop.schema.type.FloatingPoint;
import org.joda.time.LocalDate;
import org.joda.time.LocalDateTime;
import org.kitesdk.data.Format;
@@ -142,7 +143,14 @@ public class KiteDataTypeUtil {
}
}
case FLOATING_POINT:
- return Schema.Type.DOUBLE;
+ FloatingPoint fp = (FloatingPoint)column;
+ if(fp.getByteSize() <= 4L) {
+ return Schema.Type.FLOAT;
+ } else if(fp.getByteSize() <= 8L) {
+ return Schema.Type.DOUBLE;
+ } else {
+ throw new IllegalArgumentException("Unsupported size of FloatingPoint column " + fp.getByteSize());
+ }
case MAP:
return Schema.Type.MAP;
case TEXT:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
index 6e49312..a4a5b04 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.sqoop.integration.connector.hive;
-import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.test.asserts.ProviderAsserts;
import org.apache.sqoop.common.test.db.TableName;
import org.apache.sqoop.connector.common.FileFormat;
@@ -26,7 +25,6 @@ import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
-import org.apache.sqoop.test.utils.HdfsUtils;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -67,10 +65,8 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId());
// Set rdbms "FROM" config
- MConfigList fromConfig = job.getFromJobConfig();
fillRdbmsFromConfig(job, "id");
- // TODO: Kite have troubles with some data types, so we're limiting the columns to int only
- fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
+ job.getFromJobConfig().getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
// Fill the Kite "TO" config
MConfigList toConfig = job.getToJobConfig();
@@ -82,7 +78,6 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);
saveJob(job);
-
executeJob(job);
// Assert correct output
http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
index b3ec1a9..5c90501 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.sqoop.integration.connector.kite;
-import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.common.FileFormat;
import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MDriverConfig;
@@ -70,9 +69,7 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase {
// Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id");
- // TODO: Kite have troubles with some data types, so we're limiting the columns to int only
- MConfigList fromConfig = job.getFromJobConfig();
- fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
+ job.getFromJobConfig().getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
// Fill the Kite "TO" config
MConfigList toConfig = job.getToJobConfig();
@@ -84,15 +81,14 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase {
driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);
saveJob(job);
-
executeJob(job);
// Assert correct output
assertTo(
- "\"1\"",
- "\"2\"",
- "\"3\"",
- "\"4\""
+ "\"1\"",
+ "\"2\"",
+ "\"3\"",
+ "\"4\""
);
}
}