You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bo...@apache.org on 2018/02/14 14:11:53 UTC
sqoop git commit: SQOOP-2976: Flag to expand decimal values to fit
AVRO schema
Repository: sqoop
Updated Branches:
refs/heads/trunk 6984a36c5 -> f7b460b3f
SQOOP-2976: Flag to expand decimal values to fit AVRO schema
(Ferenc Szabo via Boglarka Egyed)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f7b460b3
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f7b460b3
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f7b460b3
Branch: refs/heads/trunk
Commit: f7b460b3f57c1bc81e2e0a1e8c28a331729f4213
Parents: 6984a36
Author: Boglarka Egyed <bo...@apache.org>
Authored: Wed Feb 14 15:09:19 2018 +0100
Committer: Boglarka Egyed <bo...@apache.org>
Committed: Wed Feb 14 15:09:19 2018 +0100
----------------------------------------------------------------------
src/java/org/apache/sqoop/avro/AvroUtil.java | 62 +++++++--
.../sqoop/config/ConfigurationConstants.java | 5 +
.../sqoop/mapreduce/AvroImportMapper.java | 5 +-
src/test/org/apache/sqoop/TestAvroImport.java | 13 +-
.../manager/hsqldb/TestHsqldbAvroPadding.java | 81 +++++++++++
.../oracle/OracleAvroPaddingImportTest.java | 120 ++++++++++++++++
.../SQLServerAvroPaddingImportTest.java | 134 ++++++++++++++++++
.../TestMetastoreConfigurationParameters.java | 28 ++--
.../sqoop/testutil/ArgumentArrayBuilder.java | 138 +++++++++++++++++++
.../apache/sqoop/testutil/ArgumentUtils.java | 63 ---------
.../apache/sqoop/testutil/AvroTestUtils.java | 100 ++++++++++++++
.../sqoop/testutil/BaseSqoopTestCase.java | 32 +++--
12 files changed, 665 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/java/org/apache/sqoop/avro/AvroUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/avro/AvroUtil.java b/src/java/org/apache/sqoop/avro/AvroUtil.java
index 1aae8df..caed90e 100644
--- a/src/java/org/apache/sqoop/avro/AvroUtil.java
+++ b/src/java/org/apache/sqoop/avro/AvroUtil.java
@@ -51,6 +51,9 @@ import java.util.Map;
* The service class provides methods for creating and converting Avro objects.
*/
public final class AvroUtil {
+
+ public static final String DECIMAL = "decimal";
+
public static boolean isDecimal(Schema.Field field) {
return isDecimal(field.schema());
}
@@ -65,20 +68,54 @@ public final class AvroUtil {
return false;
} else {
- return "decimal".equals(schema.getProp(LogicalType.LOGICAL_TYPE_PROP));
+ return DECIMAL.equals(schema.getProp(LogicalType.LOGICAL_TYPE_PROP));
+ }
+ }
+
+ private static BigDecimal padBigDecimal(BigDecimal bd, Schema schema) {
+ Schema schemaContainingScale = getDecimalSchema(schema);
+ if(schemaContainingScale != null) {
+ int scale = Integer.valueOf(schemaContainingScale.getObjectProp("scale").toString());
+ if (bd.scale() != scale) {
+ return bd.setScale(scale);
+ }
+ }
+ return bd;
+ }
+
+ private static Schema getDecimalSchema(Schema schema) {
+ if (schema.getType().equals(Schema.Type.UNION)) {
+ for (Schema type : schema.getTypes()) {
+ // search for decimal schema
+ Schema schemaContainingScale = getDecimalSchema(type);
+ if (schemaContainingScale != null) {
+ return schemaContainingScale;
+ }
+ }
+ } else {
+ if(DECIMAL.equals(schema.getProp(LogicalType.LOGICAL_TYPE_PROP))) {
+ return schema;
+ }
}
+ return null;
}
/**
* Convert a Sqoop's Java representation to Avro representation.
*/
- public static Object toAvro(Object o, Schema.Field field, boolean bigDecimalFormatString) {
- if (o instanceof BigDecimal && !isDecimal(field)) {
- if (bigDecimalFormatString) {
- // Returns a string representation of this without an exponent field.
- return ((BigDecimal) o).toPlainString();
- } else {
- return o.toString();
+ public static Object toAvro(Object o, Schema.Field field, boolean bigDecimalFormatString, boolean bigDecimalPaddingEnabled) {
+
+ if (o instanceof BigDecimal) {
+ if(bigDecimalPaddingEnabled) {
+ o = padBigDecimal((BigDecimal) o, field.schema());
+ }
+ if (!isDecimal(field)) {
+ if (bigDecimalFormatString) {
+ // Returns a string representation of this without an exponent field.
+ return ((BigDecimal) o).toPlainString();
+ } else {
+ return o.toString();
+ }
}
} else if (o instanceof Date) {
return ((Date) o).getTime();
@@ -136,16 +173,21 @@ public final class AvroUtil {
}
}
+ public static GenericRecord toGenericRecord(Map<String, Object> fieldMap,
+ Schema schema, boolean bigDecimalFormatString) {
+ return toGenericRecord(fieldMap, schema, bigDecimalFormatString, false);
+ }
+
/**
* Manipulate a GenericRecord instance.
*/
public static GenericRecord toGenericRecord(Map<String, Object> fieldMap,
- Schema schema, boolean bigDecimalFormatString) {
+ Schema schema, boolean bigDecimalFormatString, boolean bigDecimalPaddingEnabled) {
GenericRecord record = new GenericData.Record(schema);
for (Map.Entry<String, Object> entry : fieldMap.entrySet()) {
String avroColumn = toAvroColumn(entry.getKey());
Schema.Field field = schema.getField(avroColumn);
- Object avroObject = toAvro(entry.getValue(), field, bigDecimalFormatString);
+ Object avroObject = toAvro(entry.getValue(), field, bigDecimalFormatString, bigDecimalPaddingEnabled);
record.put(avroColumn, avroObject);
}
return record;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/java/org/apache/sqoop/config/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
index 7a19a62..2197025 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationConstants.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
@@ -106,6 +106,11 @@ public final class ConfigurationConstants {
public static final String PROP_ENABLE_AVRO_LOGICAL_TYPE_DECIMAL = "sqoop.avro.logical_types.decimal.enable";
/**
+ * Enable padding for avro logical types (decimal support only).
+ */
+ public static final String PROP_ENABLE_AVRO_DECIMAL_PADDING = "sqoop.avro.decimal_padding.enable";
+
+ /**
* The Configuration property identifying data publisher class.
*/
public static final String DATA_PUBLISH_CLASS = "sqoop.job.data.publish.class";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java b/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java
index a5e5bf5..1ce1e88 100644
--- a/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java
@@ -18,6 +18,7 @@
package org.apache.sqoop.mapreduce;
+import org.apache.sqoop.config.ConfigurationConstants;
import org.apache.sqoop.lib.LargeObjectLoader;
import org.apache.sqoop.lib.SqoopRecord;
import org.apache.avro.Schema;
@@ -44,6 +45,7 @@ public class AvroImportMapper
private Schema schema;
private LargeObjectLoader lobLoader;
private boolean bigDecimalFormatString;
+ private boolean bigDecimalPadding;
@Override
protected void setup(Context context)
@@ -54,6 +56,7 @@ public class AvroImportMapper
bigDecimalFormatString = conf.getBoolean(
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
+ bigDecimalPadding = conf.getBoolean(ConfigurationConstants.PROP_ENABLE_AVRO_DECIMAL_PADDING, false);
}
@Override
@@ -67,7 +70,7 @@ public class AvroImportMapper
throw new IOException(sqlE);
}
- GenericRecord outKey = AvroUtil.toGenericRecord(val.getFieldMap(), schema, bigDecimalFormatString);
+ GenericRecord outKey = AvroUtil.toGenericRecord(val.getFieldMap(), schema, bigDecimalFormatString, bigDecimalPadding);
wrapper.datum(outKey);
context.write(wrapper, NullWritable.get());
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/TestAvroImport.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestAvroImport.java b/src/test/org/apache/sqoop/TestAvroImport.java
index 1172fc5..2666f50 100644
--- a/src/test/org/apache/sqoop/TestAvroImport.java
+++ b/src/test/org/apache/sqoop/TestAvroImport.java
@@ -31,17 +31,14 @@ import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
import org.apache.avro.file.DataFileConstants;
import org.apache.avro.file.DataFileReader;
-import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.mapred.FsInput;
import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.testutil.AvroTestUtils;
import org.apache.sqoop.testutil.CommonArgs;
import org.apache.sqoop.testutil.HsqldbTestServer;
import org.apache.sqoop.testutil.ImportJobTestCase;
@@ -365,13 +362,7 @@ public class TestAvroImport extends ImportJobTestCase {
protected DataFileReader<GenericRecord> read(Path filename) throws IOException {
Configuration conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- FsInput fsInput = new FsInput(filename, conf);
- DatumReader<GenericRecord> datumReader =
- new GenericDatumReader<GenericRecord>();
- return new DataFileReader<GenericRecord>(fsInput, datumReader);
+ return AvroTestUtils.read(filename, conf);
}
protected void checkSchemaFile(final Schema schema) throws IOException {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/manager/hsqldb/TestHsqldbAvroPadding.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/hsqldb/TestHsqldbAvroPadding.java b/src/test/org/apache/sqoop/manager/hsqldb/TestHsqldbAvroPadding.java
new file mode 100644
index 0000000..7e42bf1
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/hsqldb/TestHsqldbAvroPadding.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.hsqldb;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sqoop.testutil.ArgumentArrayBuilder;
+import org.apache.sqoop.testutil.AvroTestUtils;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.List;
+
+
+public class TestHsqldbAvroPadding extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ TestHsqldbAvroPadding.class.getName());
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ createTestTable();
+ }
+
+ protected void createTestTable() {
+ String[] names = {"ID", "NAME", "SALARY", "DEPT"};
+ String[] types = { "INT", "VARCHAR(24)", "DECIMAL(20,5)", "VARCHAR(32)"};
+ List<String[]> inputData = AvroTestUtils.getInputData();
+ createTableWithColTypesAndNames(names, types, new String[0]);
+ insertIntoTable(names, types, inputData.get(0));
+ insertIntoTable(names, types, inputData.get(1));
+ insertIntoTable(names, types, inputData.get(2));
+ }
+
+ protected ArgumentArrayBuilder getArgumentArrayBuilder() {
+ ArgumentArrayBuilder builder = AvroTestUtils.getBuilderForAvroPaddingTest(this);
+ builder.withOption("connect", getConnectString());
+ return builder;
+ }
+
+ @Test
+ public void testAvroImportWithoutPaddingFails() throws IOException {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Failure during job; return status 1");
+ String[] args = getArgumentArrayBuilder().build();
+ runImport(args);
+ }
+
+ @Test
+ public void testAvroImportWithPadding() throws IOException {
+ ArgumentArrayBuilder builder = getArgumentArrayBuilder();
+ builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
+ String[] args = builder.build();
+ runImport(args);
+ AvroTestUtils.verify(AvroTestUtils.getExpectedResults(), getConf(), getTablePath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/manager/oracle/OracleAvroPaddingImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleAvroPaddingImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleAvroPaddingImportTest.java
new file mode 100644
index 0000000..f217f0b
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleAvroPaddingImportTest.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sqoop.manager.oracle;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.ArgumentArrayBuilder;
+import org.apache.sqoop.testutil.AvroTestUtils;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+
+ public class OracleAvroPaddingImportTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleAvroPaddingImportTest.class.getName());
+
+ private Configuration conf = new Configuration();
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Override
+ protected Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String getConnectString() {
+ return org.apache.sqoop.manager.oracle.util.OracleUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ org.apache.sqoop.manager.oracle.util.OracleUtils.setOracleAuth(opts);
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ OracleUtils.dropTable(table, getManager());
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ String [] names = {"ID", "NAME", "SALARY", "DEPT"};
+ String [] types = { "INT", "VARCHAR(24)", "DECIMAL(20,5)", "VARCHAR(32)"};
+ List<String[]> inputData = AvroTestUtils.getInputData();
+ createTableWithColTypesAndNames(names, types, new String[0]);
+ insertIntoTable(names, types, inputData.get(0));
+ insertIntoTable(names, types, inputData.get(1));
+ insertIntoTable(names, types, inputData.get(2));
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException e) {
+ LOG.warn("Error trying to drop table on tearDown: " + e);
+ }
+ super.tearDown();
+ }
+
+ protected ArgumentArrayBuilder getArgsBuilder() {
+ ArgumentArrayBuilder builder = AvroTestUtils.getBuilderForAvroPaddingTest(this);
+ builder.withOption("connect", getConnectString());
+ return builder;
+ }
+
+ @Test
+ public void testAvroImportWithoutPaddingFails() throws IOException {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Failure during job; return status 1");
+ String[] args = getArgsBuilder().build();
+ runImport(args);
+ }
+
+ @Test
+ public void testAvroImportWithPadding() throws IOException {
+ ArgumentArrayBuilder builder = getArgsBuilder();
+ builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
+ runImport(builder.build());
+ AvroTestUtils.verify(AvroTestUtils.getExpectedResults(), getConf(), getTablePath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/manager/sqlserver/SQLServerAvroPaddingImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerAvroPaddingImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerAvroPaddingImportTest.java
new file mode 100644
index 0000000..27dc0cd
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerAvroPaddingImportTest.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.avro.AvroUtil;
+import org.apache.sqoop.testutil.ArgumentArrayBuilder;
+import org.apache.sqoop.testutil.AvroTestUtils;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+
+public class SQLServerAvroPaddingImportTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerAvroPaddingImportTest.class.getName());
+
+ private Configuration conf = new Configuration();
+
+ @Override
+ protected String getConnectString() {
+ return MSSQLTestUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions options = new SqoopOptions();
+ options.setConnectString(MSSQLTestUtils.CONNECT_STRING);
+ options.setUsername(MSSQLTestUtils.DATABASE_USER);
+ options.setPassword(MSSQLTestUtils.DATABASE_PASSWORD);
+ return options;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String dropTableIfExistsCommand(String table) {
+ return "DROP TABLE IF EXISTS " + manager.escapeTableName(table);
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ String [] names = {"ID", "NAME", "SALARY", "DEPT"};
+ String [] types = { "INT", "VARCHAR(24)", "DECIMAL(20,5)", "VARCHAR(32)"};
+ List<String[]> inputData = AvroTestUtils.getInputData();
+ createTableWithColTypesAndNames(names, types, new String[0]);
+ insertIntoTable(names, types, inputData.get(0));
+ insertIntoTable(names, types, inputData.get(1));
+ insertIntoTable(names, types, inputData.get(2));
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException e) {
+ LOG.warn("Error trying to drop table on tearDown: " + e);
+ }
+ super.tearDown();
+ }
+
+ protected ArgumentArrayBuilder getArgsBuilder() {
+ ArgumentArrayBuilder builder = AvroTestUtils.getBuilderForAvroPaddingTest(this);
+ builder.withOption("connect", MSSQLTestUtils.CONNECT_STRING);
+ builder.withOption("username", MSSQLTestUtils.DATABASE_USER);
+ builder.withOption("password", MSSQLTestUtils.DATABASE_PASSWORD);
+ return builder;
+ }
+
+ /**
+ * Test for avro import with a number value in the table.
+ * SQL Server stores the values padded in the database, therefore this import should always be successful
+ * (Oracle for instance doesn't pad numbers in the database, therefore that one fails without the
+ * sqoop.avro.decimal_padding.enable property)
+ * @throws IOException
+ */
+ @Test
+ public void testAvroImportWithoutPaddingFails() throws IOException {
+ String[] args = getArgsBuilder().build();
+ runImport(args);
+ String [] expectedResults = AvroTestUtils.getExpectedResults();
+ AvroTestUtils.verify(expectedResults, getConf(), getTablePath());
+ }
+
+ /**
+ * This test covers a different code path than {@link #testAvroImportWithoutPaddingFails()},
+ * since the BigDecimal values are checked and padded by Sqoop in
+ * {@link AvroUtil#padBigDecimal(java.math.BigDecimal, org.apache.avro.Schema)}
+ * No actual padding occurs, as the values coming back from SQL Server are already padded with 0s.
+ * @throws IOException
+ */
+ @Test
+ public void testAvroImportWithPadding() throws IOException {
+ ArgumentArrayBuilder builder = getArgsBuilder();
+ builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
+ runImport(builder.build());
+ String [] expectedResults = AvroTestUtils.getExpectedResults();
+ AvroTestUtils.verify(expectedResults, getConf(), getTablePath());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/metastore/TestMetastoreConfigurationParameters.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/metastore/TestMetastoreConfigurationParameters.java b/src/test/org/apache/sqoop/metastore/TestMetastoreConfigurationParameters.java
index 391dc33..0f1eb89 100644
--- a/src/test/org/apache/sqoop/metastore/TestMetastoreConfigurationParameters.java
+++ b/src/test/org/apache/sqoop/metastore/TestMetastoreConfigurationParameters.java
@@ -18,9 +18,9 @@
package org.apache.sqoop.metastore;
+import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.HsqldbTestServer;
import org.apache.sqoop.Sqoop;
-import org.apache.sqoop.testutil.Argument;
import org.apache.sqoop.tool.JobTool;
import org.junit.AfterClass;
import org.junit.Before;
@@ -33,12 +33,6 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import static java.util.Arrays.asList;
-import static java.util.Collections.singleton;
-import static org.apache.sqoop.testutil.Argument.from;
-import static org.apache.sqoop.testutil.Argument.fromPair;
-import static org.apache.sqoop.testutil.ArgumentUtils.createArgumentArray;
-import static org.apache.sqoop.testutil.ArgumentUtils.createArgumentArrayFromProperties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -74,8 +68,9 @@ public class TestMetastoreConfigurationParameters {
@Test
public void testJobToolWithAutoConnectDisabledFails() throws IOException {
- Argument autoConnectProperty = fromPair("sqoop.metastore.client.enable.autoconnect", "false");
- String[] arguments = createArgumentArrayFromProperties(singleton(autoConnectProperty));
+ ArgumentArrayBuilder builder = new ArgumentArrayBuilder()
+ .withProperty("sqoop.metastore.client.enable.autoconnect", "false");
+ String[] arguments = builder.build();
assertEquals(STATUS_FAILURE, Sqoop.runSqoop(sqoop, arguments));
}
@@ -92,15 +87,12 @@ public class TestMetastoreConfigurationParameters {
}
private int runJobToolWithAutoConnectUrlAndCorrectUsernamePasswordSpecified() {
- Argument url = fromPair("sqoop.metastore.client.autoconnect.url", HsqldbTestServer.getUrl());
- Argument user = fromPair("sqoop.metastore.client.autoconnect.username", TEST_USER);
- Argument password = fromPair("sqoop.metastore.client.autoconnect.password", TEST_PASSWORD);
- Argument listJob = from("list");
-
- Iterable<Argument> properties = asList(url, user, password);
- Iterable<Argument> options = singleton(listJob);
-
- String[] arguments = createArgumentArray(properties, options);
+ ArgumentArrayBuilder builder = new ArgumentArrayBuilder()
+ .withProperty("sqoop.metastore.client.autoconnect.url", HsqldbTestServer.getUrl())
+ .withProperty("sqoop.metastore.client.autoconnect.username", TEST_USER)
+ .withProperty("sqoop.metastore.client.autoconnect.password", TEST_PASSWORD)
+ .withOption("list");
+ String[] arguments = builder.build();
return Sqoop.runSqoop(sqoop, arguments);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/testutil/ArgumentArrayBuilder.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/ArgumentArrayBuilder.java b/src/test/org/apache/sqoop/testutil/ArgumentArrayBuilder.java
new file mode 100644
index 0000000..00ce4fe
--- /dev/null
+++ b/src/test/org/apache/sqoop/testutil/ArgumentArrayBuilder.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.testutil;
+
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+
+public class ArgumentArrayBuilder {
+
+ private static final String PROPERTY_PREFIX = "-D";
+
+ private static final String OPTION_PREFIX = "--";
+ public static final String TOOL_ARG_SEPARATOR = "--";
+
+ private List<Argument> properties;
+
+ private List<Argument> options;
+
+ private List<Argument> toolOptions;
+
+ private boolean withCommonHadoopFlags;
+
+ public ArgumentArrayBuilder() {
+ properties = new ArrayList<>();
+ options = new ArrayList<>();
+ toolOptions = new ArrayList<>();
+ }
+
+ public ArgumentArrayBuilder withProperty(String name, String value) {
+ properties.add(new Argument(name, value));
+ return this;
+ }
+
+ public ArgumentArrayBuilder withProperty(String name) {
+ properties.add(new Argument(name));
+ return this;
+ }
+
+ public ArgumentArrayBuilder withOption(String name, String value) {
+ options.add(new Argument(name, value));
+ return this;
+ }
+
+ public ArgumentArrayBuilder withOption(String name) {
+ options.add(new Argument(name));
+ return this;
+ }
+
+ public ArgumentArrayBuilder withToolOption(String name, String value) {
+ toolOptions.add(new Argument(name, value));
+ return this;
+ }
+
+ public ArgumentArrayBuilder withToolOption(String name) {
+ toolOptions.add(new Argument(name));
+ return this;
+ }
+
+ public ArgumentArrayBuilder with(ArgumentArrayBuilder otherBuilder) {
+ properties.addAll(otherBuilder.properties);
+ options.addAll(otherBuilder.options);
+ return this;
+ }
+
+ public ArgumentArrayBuilder withCommonHadoopFlags(boolean b) {
+ withCommonHadoopFlags = b;
+ return this;
+ }
+
+ public ArgumentArrayBuilder withCommonHadoopFlags() {
+ withCommonHadoopFlags = true;
+ return this;
+ }
+
+ /**
+ * Transforms the given options, properties and toolOptions to the command line format Sqoop expects,
+ * by adding dashes (--) and the capital D letter when it's necessary (in front of properties)
+ * @return String array that can be used to run tests
+ */
+ public String[] build() {
+ List<String> result = new ArrayList<>();
+ if (withCommonHadoopFlags) {
+ CommonArgs.addHadoopFlags(result);
+ }
+ if (CollectionUtils.isNotEmpty(properties)) {
+ Collections.addAll(result, createArgumentArrayFromProperties(properties));
+ }
+ if (CollectionUtils.isNotEmpty(options)) {
+ Collections.addAll(result, createArgumentArrayFromOptions(options));
+ }
+ if (CollectionUtils.isNotEmpty(toolOptions)) {
+ result.add(TOOL_ARG_SEPARATOR);
+ Collections.addAll(result, createArgumentArrayFromOptions(toolOptions));
+ }
+ return result.toArray(new String[result.size()]);
+ }
+
+ private String[] createArgumentArrayFromProperties(List<Argument> properties) {
+ List<String> result = new ArrayList<>();
+ for (Argument property : properties) {
+ result.add(PROPERTY_PREFIX);
+ result.add(property.toString());
+ }
+ return result.toArray(new String[result.size()]);
+ }
+
+ private String[] createArgumentArrayFromOptions(List<Argument> options) {
+ List<String> result = new ArrayList<>();
+ for (Argument option : options) {
+ result.add(OPTION_PREFIX + option.getName());
+ if (!isEmpty(option.getValue())) {
+ result.add(option.getValue());
+ }
+ }
+ return result.toArray(new String[result.size()]);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/testutil/ArgumentUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/ArgumentUtils.java b/src/test/org/apache/sqoop/testutil/ArgumentUtils.java
deleted file mode 100644
index 2f95e45..0000000
--- a/src/test/org/apache/sqoop/testutil/ArgumentUtils.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.testutil;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.commons.lang3.StringUtils.isEmpty;
-
-public final class ArgumentUtils {
-
- private static final String PROPERTY_PREFIX = "-D";
-
- private static final String OPTION_PREFIX = "--";
-
- public static String[] createArgumentArrayFromProperties(Iterable<Argument> properties) {
- List<String> result = new ArrayList<>();
- for (Argument property : properties) {
- result.add(PROPERTY_PREFIX);
- result.add(property.toString());
- }
-
- return result.toArray(new String[result.size()]);
- }
-
- public static String[] createArgumentArrayFromOptions(Iterable<Argument> options) {
- List<String> result = new ArrayList<>();
- for (Argument option : options) {
- result.add(OPTION_PREFIX + option.getName());
- if (!isEmpty(option.getValue())) {
- result.add(option.getValue());
- }
- }
-
- return result.toArray(new String[result.size()]);
- }
-
- public static String[] createArgumentArray(Iterable<Argument> properties, Iterable<Argument> options) {
- List<String> result = new ArrayList<>();
- Collections.addAll(result, createArgumentArrayFromProperties(properties));
- Collections.addAll(result, createArgumentArrayFromOptions(options));
-
- return result.toArray(new String[result.size()]);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/testutil/AvroTestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/AvroTestUtils.java b/src/test/org/apache/sqoop/testutil/AvroTestUtils.java
new file mode 100644
index 0000000..75940bf
--- /dev/null
+++ b/src/test/org/apache/sqoop/testutil/AvroTestUtils.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.testutil;
+
+import org.apache.avro.Conversions;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.mapred.FsInput;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertEquals;
+
+public class AvroTestUtils {
+
+ public static List<String[]> getInputData() {
+ List<String[]> data = new ArrayList<>();
+ data.add(new String[]{"1", "'Aaron'", "1000000.05", "'engineering'"});
+ data.add(new String[]{"2", "'Bob'", "400.10", "'sales'"});
+ data.add(new String[]{"3", "'Fred'", "15.23", "'marketing'"});
+ return data;
+ }
+
+ public static String[] getExpectedResults() {
+ return new String[] {
+ "{\"ID\": 1, \"NAME\": \"Aaron\", \"SALARY\": 1000000.05000, \"DEPT\": \"engineering\"}",
+ "{\"ID\": 2, \"NAME\": \"Bob\", \"SALARY\": 400.10000, \"DEPT\": \"sales\"}",
+ "{\"ID\": 3, \"NAME\": \"Fred\", \"SALARY\": 15.23000, \"DEPT\": \"marketing\"}"
+ };
+ }
+
+ public static ArgumentArrayBuilder getBuilderForAvroPaddingTest(BaseSqoopTestCase testCase) {
+ ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
+ return builder.withCommonHadoopFlags(true)
+ .withProperty("sqoop.avro.logical_types.decimal.enable", "true")
+ .withOption("as-avrodatafile")
+ .withOption("warehouse-dir", testCase.getWarehouseDir())
+ .withOption("num-mappers", "1")
+ .withOption("table", testCase.getTableName());
+ }
+
+ public static void verify(String[] expectedResults, Configuration conf, Path tablePath) {
+ Path outputFile = new Path(tablePath, "part-m-00000.avro");
+ GenericData.get().addLogicalTypeConversion(new Conversions.DecimalConversion());
+ try (DataFileReader<GenericRecord> reader = read(outputFile, conf)) {
+ GenericRecord record;
+ if (!reader.hasNext() && expectedResults != null && expectedResults.length > 0) {
+ fail("empty file was not expected");
+ }
+ int i = 0;
+ while (reader.hasNext()){
+ record = reader.next();
+ assertEquals(expectedResults[i++], record.toString());
+ }
+ }
+ catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ /**
+ * Return an instance of DataFileReader for the given filename.
+ * @param filename path that we're opening a reader for.
+ * @param conf
+ * @return instance of DataFileReader.
+ * @throws IOException
+ */
+ public static DataFileReader<GenericRecord> read(Path filename, Configuration conf) throws IOException {
+ if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
+ conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
+ }
+ FsInput fsInput = new FsInput(filename, conf);
+ DatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
+ return new DataFileReader<>(fsInput, datumReader);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f7b460b3/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java b/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java
index 588f439..a5f85a0 100644
--- a/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java
+++ b/src/test/org/apache/sqoop/testutil/BaseSqoopTestCase.java
@@ -167,7 +167,7 @@ public abstract class BaseSqoopTestCase {
// instance variables populated during setUp, used during tests
private HsqldbTestServer testServer;
- private ConnManager manager;
+ protected ConnManager manager;
private static boolean isLog4jConfigured = false;
@@ -299,7 +299,8 @@ public abstract class BaseSqoopTestCase {
*/
protected void dropTableIfExists(String table) throws SQLException {
Connection conn = getManager().getConnection();
- PreparedStatement statement = conn.prepareStatement(dropTableIfExistsCommand(table),
+ String dropStatement = dropTableIfExistsCommand(table);
+ PreparedStatement statement = conn.prepareStatement(dropStatement,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
try {
statement.executeUpdate();
@@ -351,7 +352,6 @@ public abstract class BaseSqoopTestCase {
columnDefStr += ", ";
}
}
-
createTableStr = "CREATE TABLE " + manager.escapeTableName(newTableName) + "(" + columnDefStr + ")";
LOG.info("Creating table: " + createTableStr);
statement = conn.prepareStatement(
@@ -423,22 +423,28 @@ public abstract class BaseSqoopTestCase {
}
}
- /**
- * insert into a table with a set of columns values for a given row.
- * @param colTypes the types of the columns to make
- * @param vals the SQL text for each value to insert
- */
protected void insertIntoTable(String[] colTypes, String[] vals) {
- assert colNames != null;
- assert colNames.length == vals.length;
+ insertIntoTable(null, colTypes, vals);
+ }
+
+ protected void insertIntoTable(String[] columns, String[] colTypes, String[] vals) {
+ assert colTypes != null;
+ assert colTypes.length == vals.length;
Connection conn = null;
PreparedStatement statement = null;
- String[] colNames = new String[vals.length];
- for( int i = 0; i < vals.length; i++) {
- colNames[i] = BASE_COL_NAME + Integer.toString(i);
+ String[] colNames;
+ if (columns == null){
+ colNames = new String[vals.length];
+ for( int i = 0; i < vals.length; i++) {
+ colNames[i] = BASE_COL_NAME + Integer.toString(i);
+ }
}
+ else {
+ colNames = columns;
+ }
+
try {
conn = getManager().getConnection();
for (int count=0; vals != null && count < vals.length/colTypes.length;