You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2017/07/06 09:21:16 UTC
sqoop git commit: SQOOP-3206: Make sqoop fail if user uses --direct
connector and tries to encode a null value when using a MySQL database
Repository: sqoop
Updated Branches:
refs/heads/trunk af277ae3e -> 2c199a7f4
SQOOP-3206: Make sqoop fail if user uses --direct connector and tries to encode a null value when using a MySQL database
(Zach Berkowitz via Anna Szonyi)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2c199a7f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2c199a7f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2c199a7f
Branch: refs/heads/trunk
Commit: 2c199a7f4fc744f653f036d5afb6f49b0d4c445c
Parents: af277ae
Author: Anna Szonyi <an...@apache.org>
Authored: Thu Jun 29 18:44:14 2017 +0200
Committer: Anna Szonyi <an...@apache.org>
Committed: Thu Jun 29 18:44:14 2017 +0200
----------------------------------------------------------------------
src/java/org/apache/sqoop/tool/ExportTool.java | 14 ++++++++++++
src/java/org/apache/sqoop/tool/ImportTool.java | 11 +++++++--
.../sqoop/manager/DirectMySQLExportTest.java | 18 +++++++++++++++
.../cloudera/sqoop/manager/DirectMySQLTest.java | 24 ++++++++++++++------
.../tool/TestExportToolValidateOptions.java | 16 +++++++++++++
.../sqoop/tool/TestValidateImportOptions.java | 17 +++++++++++++-
6 files changed, 90 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/java/org/apache/sqoop/tool/ExportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java
index 5512fa7..cd6cdf3 100644
--- a/src/java/org/apache/sqoop/tool/ExportTool.java
+++ b/src/java/org/apache/sqoop/tool/ExportTool.java
@@ -33,6 +33,7 @@ import com.cloudera.sqoop.cli.RelatedOptions;
import com.cloudera.sqoop.cli.ToolOptions;
import com.cloudera.sqoop.manager.ExportJobContext;
import com.cloudera.sqoop.util.ExportException;
+import static org.apache.sqoop.manager.SupportedManagers.MYSQL;
/**
* Tool that performs HDFS exports to databases.
@@ -385,8 +386,21 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
void vaildateDirectExportOptions(SqoopOptions options) throws InvalidOptionsException {
if (options.isDirect()) {
validateHasDirectConnectorOption(options);
+ validateDirectMysqlOptions(options);
}
}
+
+ public void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException {
+ if (!MYSQL.isTheManagerTypeOf(options)) {
+ return;
+ }
+ if (options.getInNullStringValue() != null || options.getInNullNonStringValue() != null) {
+ throw new InvalidOptionsException(
+ "The --direct option is not compatible with the --input-null-string or " +
+ "--input-null-non-string command for MySQL exports");
+ }
+ }
+
private void applyNewUpdateOptions(CommandLine in, SqoopOptions out)
throws InvalidOptionsException {
if (in.hasOption(UPDATE_MODE_ARG)) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 4b1b12d..78c7758 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -1120,13 +1120,20 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
}
void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException {
- if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile
- && MYSQL.isTheManagerTypeOf(options)) {
+ if (!MYSQL.isTheManagerTypeOf(options)) {
+ return;
+ }
+ if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
throw new InvalidOptionsException(
"MySQL direct import currently supports only text output format. "
+ "Parameters --as-sequencefile --as-avrodatafile and --as-parquetfile are not "
+ "supported with --direct params in MySQL case.");
}
+ if (options.getNullStringValue() != null || options.getNullNonStringValue() != null) {
+ throw new InvalidOptionsException(
+ "The --direct option is not compatible with the --null-string or " +
+ "--null-non-string command for MySQL imports");
+ }
}
/**
* Validate the incremental import options.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
index 9fa8816..f9e3cde 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
@@ -303,6 +303,24 @@ public class DirectMySQLExportTest extends TestExport {
}
}
+ @Test(expected = IOException.class)
+ public void testExportInputNullStringFailsValidate() throws IOException {
+ runExport(getArgv(true, 10, 10,
+ "--username", MySQLAuthTest.AUTH_TEST_USER,
+ "--password", MySQLAuthTest.AUTH_TEST_PASS,
+ "--connect", MySQLAuthTest.AUTH_CONNECT_STRING,
+ "--input-null-string", "null"));
+ }
+
+ @Test(expected = IOException.class)
+ public void testExportInputNullNonStringFailsValidate() throws IOException {
+ runExport(getArgv(true, 10, 10,
+ "--username", MySQLAuthTest.AUTH_TEST_USER,
+ "--password", MySQLAuthTest.AUTH_TEST_PASS,
+ "--connect", MySQLAuthTest.AUTH_CONNECT_STRING,
+ "--input-null-non-string", "null"));
+ }
+
@Ignore("Ignoring this test as staging is not supported in direct mode.")
@Override
@Test
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
index a58fa17..247ce0b 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
@@ -192,13 +192,7 @@ public class DirectMySQLTest extends ImportJobTestCase {
}
String [] argv = getArgv(mysqlOutputDelims, isDirect, tableName, extraArgs);
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
+ runImport(argv);
File f = new File(filePath.toString());
assertTrue("Could not find imported data file: " + f, f.exists());
@@ -358,6 +352,22 @@ public class DirectMySQLTest extends ImportJobTestCase {
}
+ @Test(expected = IOException.class)
+ public void testSqoopNullStringValueFailsValidate() throws Exception {
+ String [] expectedResults = {};
+ String [] extraArgs = {"--null-string", "abc"};
+
+ doImport(false, true, getTableName(), expectedResults, extraArgs);
+ }
+
+ @Test(expected = IOException.class)
+ public void testSqoopNullNonStringValueFailsValidate() throws Exception {
+ String [] expectedResults = {};
+ String [] extraArgs = {"--null-non-string", "abc"};
+
+ doImport(false, true, getTableName(), expectedResults, extraArgs);
+ }
+
@Test
public void testJdbcEscapedColumnName() throws Exception {
// Test a JDBC-based import of a table with a column whose name is
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
index dfe1952..0018fb1 100644
--- a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
+++ b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java
@@ -66,6 +66,22 @@ public class TestExportToolValidateOptions {
exportTool.vaildateDirectExportOptions(options);
}
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportInputNullStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getInNullNonStringValue()).thenReturn("abc");
+
+ exportTool.validateDirectMysqlOptions(options);
+ }
+
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportInputNullNonStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getInNullNonStringValue()).thenReturn("abc");
+
+ exportTool.validateDirectMysqlOptions(options);
+ }
+
private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
return stubOptions(supportedManagers, true);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
index acf4fcf..d4084ed 100644
--- a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
+++ b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java
@@ -44,7 +44,6 @@ public class TestValidateImportOptions {
when(options.getConnectString()).thenReturn(mysqlConnectionString);
importTool.validateDirectMysqlOptions(options);
verify(options, times(1)).getFileLayout();
- verifyNoMoreInteractions(options);
}
@@ -107,6 +106,22 @@ public class TestValidateImportOptions {
importTool.validateDirectImportOptions(options);
}
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNullNonStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getNullNonStringValue()).thenReturn("abc");
+
+ importTool.validateDirectMysqlOptions(options);
+ }
+
+ @Test(expected = SqoopOptions.InvalidOptionsException.class)
+ public void givenDirectImportNullStringThrows() throws SqoopOptions.InvalidOptionsException {
+ SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL);
+ when(options.getNullStringValue()).thenReturn("abc");
+
+ importTool.validateDirectMysqlOptions(options);
+ }
+
private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) {
return stubOptions(supportedManagers, true);
}