You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ma...@apache.org on 2017/03/20 15:41:14 UTC
sqoop git commit: SQOOP-3152: Fix SqoopOptions#parseColumnMapping,
thus Sqoop will support mappings with precisison (e.g. DECIMAL(10,
5)) in the future properly
Repository: sqoop
Updated Branches:
refs/heads/trunk 6d8a670d3 -> 803edc279
SQOOP-3152: Fix SqoopOptions#parseColumnMapping, thus Sqoop will support
mappings with precisison (e.g. DECIMAL(10,5)) in the future properly
(Eric Lin via Attila Szabo)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/803edc27
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/803edc27
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/803edc27
Branch: refs/heads/trunk
Commit: 803edc27925ec6ee950b2e7fa2ffea5727248fa6
Parents: 6d8a670
Author: Attila Szabo <ma...@apache.org>
Authored: Mon Mar 20 16:37:49 2017 +0100
Committer: Attila Szabo <ma...@apache.org>
Committed: Mon Mar 20 16:37:49 2017 +0100
----------------------------------------------------------------------
src/docs/web/images/Thumbs.db | Bin 0 -> 4608 bytes
src/java/org/apache/sqoop/SqoopOptions.java | 9 +++--
.../com/cloudera/sqoop/hive/TestHiveImport.java | 17 ++++++++
.../cloudera/sqoop/hive/TestTableDefWriter.java | 39 ++++++++++++++++++-
testdata/hive/scripts/decimalMapImport.q | 17 ++++++++
5 files changed, 78 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/docs/web/images/Thumbs.db
----------------------------------------------------------------------
diff --git a/src/docs/web/images/Thumbs.db b/src/docs/web/images/Thumbs.db
new file mode 100644
index 0000000..bc9428c
Binary files /dev/null and b/src/docs/web/images/Thumbs.db differ
http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index b33b54b..caf95f6 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -1322,11 +1322,14 @@ public class SqoopOptions implements Cloneable {
public void setPasswordAlias(String alias) {
this.passwordAlias = alias;
}
+
protected void parseColumnMapping(String mapping,
Properties output) {
output.clear();
- String[] maps = mapping.split(",");
+ // replace (xx,xx) with (xx#xx), so that we can just split by "," afterwards
+ String[] maps = mapping.replaceAll("\\(([0-9]+),([0-9]+)\\)", "($1#$2)").split(",");
+
for(String map : maps) {
String[] details = map.split("=");
if (details.length != 2) {
@@ -1336,8 +1339,8 @@ public class SqoopOptions implements Cloneable {
try {
output.put(
- URLDecoder.decode(details[0], "UTF-8"),
- URLDecoder.decode(details[1], "UTF-8"));
+ URLDecoder.decode(details[0].replace("#", ","), "UTF-8"),
+ URLDecoder.decode(details[1].replace("#", ","), "UTF-8"));
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException("Encoding not supported. "
+ "Column mapping should be UTF-8 encoding.");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
index 6f13fe2..33e0cc4 100644
--- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
+++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
@@ -499,6 +499,23 @@ public class TestHiveImport extends ImportJobTestCase {
getArgv(false, null), new ImportTool());
}
+ /** Test that DECIMALS using --map-column-hive option maps can run without issues. */
+ @Test
+ public void testDecimalMapColumnHive() throws IOException {
+ final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT";
+ setCurTableName(TABLE_NAME);
+ setNumCols(2);
+ String [] types = { "NUMERIC", "CHAR(64)" };
+ String [] vals = { "12343.14159", "'foo'" };
+
+ ArrayList<String> args = new ArrayList<String>();
+ args.add("--map-column-hive");
+ args.add(BASE_COL_NAME + "0=DECIMAL(10,10)");
+
+ runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q",
+ getArgv(false, args.toArray(new String[args.size()])), new ImportTool());
+ }
+
/** If bin/hive returns an error exit status, we should get an IOException. */
@Test
public void testHiveExitFails() throws IOException {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
index 6af12da..dbf0dde 100644
--- a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
+++ b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
@@ -180,7 +180,7 @@ public class TestTableDefWriter {
}
@Test
- public void testUserMapping() throws Exception {
+ public void testUserMappingNoDecimal() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING,value=INTEGER",
};
@@ -207,6 +207,43 @@ public class TestTableDefWriter {
}
@Test
+ public void testUserMappingWithDecimal() throws Exception {
+ String[] args = {
+ "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," +
+ "value3=DECIMAL(4,5),value4=VARCHAR(255)",
+ };
+ Configuration conf = new Configuration();
+ SqoopOptions options =
+ new ImportTool().parseArguments(args, null, null, false);
+ TableDefWriter writer = new TableDefWriter(options,
+ null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
+
+ Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
+ colTypes.put("id", Types.INTEGER);
+ colTypes.put("value1", Types.VARCHAR);
+ colTypes.put("value2", Types.DOUBLE);
+ colTypes.put("value3", Types.FLOAT);
+ colTypes.put("value4", Types.CHAR);
+ writer.setColumnTypes(colTypes);
+
+ String createTable = writer.getCreateTableStmt();
+
+ assertNotNull(createTable);
+
+ assertTrue(createTable.contains("`id` STRING"));
+ assertTrue(createTable.contains("`value1` INTEGER"));
+ assertTrue(createTable.contains("`value2` DECIMAL(13,5)"));
+ assertTrue(createTable.contains("`value3` DECIMAL(4,5)"));
+ assertTrue(createTable.contains("`value4` VARCHAR(255)"));
+
+ assertFalse(createTable.contains("`id` INTEGER"));
+ assertFalse(createTable.contains("`value1` STRING"));
+ assertFalse(createTable.contains("`value2` DOUBLE"));
+ assertFalse(createTable.contains("`value3` FLOAT"));
+ assertFalse(createTable.contains("`value4` CHAR"));
+ }
+
+ @Test
public void testUserMappingFailWhenCantBeApplied() throws Exception {
String[] args = {
"--map-column-hive", "id=STRING,value=INTEGER",
http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/testdata/hive/scripts/decimalMapImport.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/decimalMapImport.q b/testdata/hive/scripts/decimalMapImport.q
new file mode 100644
index 0000000..8f05d58
--- /dev/null
+++ b/testdata/hive/scripts/decimalMapImport.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;