You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ma...@apache.org on 2017/02/15 08:35:02 UTC
sqoop git commit: SQOOP-3061: Improve query validation in connection
with --options-file thus correct options/queries like "--query SELECT * FROM
test WHERE a = 'b'" will not fail with "Malformed option in options file"
error message
Repository: sqoop
Updated Branches:
refs/heads/trunk 98c5ccb80 -> 24d08185c
SQOOP-3061: Improve query validation in connection with --options-file
thus correct options/queries like "--query SELECT * FROM test WHERE a = 'b'"
will not fail with "Malformed option in options file" error message
(Eric Lin via Attila Szabo)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/24d08185
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/24d08185
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/24d08185
Branch: refs/heads/trunk
Commit: 24d08185c502e416265f1fa78300347ad02a4e1f
Parents: 98c5ccb
Author: Attila Szabo <ma...@apache.org>
Authored: Wed Feb 15 09:30:16 2017 +0100
Committer: Attila Szabo <ma...@apache.org>
Committed: Wed Feb 15 09:30:16 2017 +0100
----------------------------------------------------------------------
.../org/apache/sqoop/util/OptionsFileUtil.java | 24 ++-
.../sqoop/util/TestOptionsFileExpansion.java | 156 +++++++++++++++++++
2 files changed, 173 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/24d08185/src/java/org/apache/sqoop/util/OptionsFileUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/OptionsFileUtil.java b/src/java/org/apache/sqoop/util/OptionsFileUtil.java
index c476e00..993ac1b 100644
--- a/src/java/org/apache/sqoop/util/OptionsFileUtil.java
+++ b/src/java/org/apache/sqoop/util/OptionsFileUtil.java
@@ -24,7 +24,9 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -147,10 +149,10 @@ public final class OptionsFileUtil {
* Removes the surrounding quote characters from the given string. The quotes
* are identified by the quote parameter, the given string by option. The
* fileName parameter is used for raising exceptions with relevant message.
- * @param fileName
- * @param option
- * @param quote
- * @return
+ * @param fileName String The name of the file that contains sqoop options
+ * @param option String the actual options in the options file
+ * @param quote char The quote that we need to validate on, either single or double quotes
+ * @return String The validated options string
* @throws Exception
*/
private static String removeQuoteCharactersIfNecessary(String fileName,
@@ -167,11 +169,19 @@ public final class OptionsFileUtil {
}
if (startingQuote || endingQuote) {
- throw new Exception("Malformed option in options file("
- + fileName + "): " + option);
+ // Regular expression looks like below:
+ // .*=\s*".*"$ OR .*=\s*'.*'$
+ // it tries to match the following:
+ // .... column_name = "values" OR .... column_name = 'values'
+ // so that the query like:
+ // SELECT * FROM table WHERE column = "values"
+ // is valid even though it ends with double quote but no starting double quote
+ if (!Pattern.matches(".*=\\s*"+quote+".*"+quote+"$", option)) {
+ throw new Exception("Malformed option in options file("
+ + fileName + "): " + option);
+ }
}
return option;
}
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/24d08185/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java
index 6d3f0f3..d403f3b 100644
--- a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java
+++ b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java
@@ -165,6 +165,162 @@ public class TestOptionsFileExpansion {
}
}
+ @Test
+ public void testValidFreeFormQueryNoQuotes() throws Exception {
+ String[] input = new String[]{
+ "--query",
+ "SELECT * FROM table",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "SELECT * FROM table",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQuerySingleQuotesStartAndEnd() throws Exception {
+ String[] input = new String[]{
+ "--query",
+ "'SELECT * FROM table'",
+ };
+
+ String[] output = new String[]{
+ "--query",
+ "SELECT * FROM table",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQueryDoubleQuotesStartAndEnd() throws Exception {
+ String[] input = new String[]{
+ "--query",
+ "\"SELECT * FROM table\"",
+ };
+
+ String[] output = new String[]{
+ "--query",
+ "SELECT * FROM table",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQuerySingleQuotesInWhere() throws Exception {
+ String[] input = new String[]{
+ "--query",
+ "SELECT * FROM table WHERE a = '1'",
+ };
+
+ String[] output = new String[]{
+ "--query",
+ "SELECT * FROM table WHERE a = '1'",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQuerySingleAndDoubleQuotesInWhere() throws Exception {
+ String[] input = new String[] {
+ "--query",
+ "SELECT * FROM table WHERE a = '1' AND b = \"testing\"",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "SELECT * FROM table WHERE a = '1' AND b = \"testing\"",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQueryQuotesInTableNameAndColumnName() throws Exception {
+ String[] input = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = 'a'",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = 'a'",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQueryQuotesInTableNameAndColumnName2() throws Exception {
+ String[] input = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = 'a\"'",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = 'a\"'",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQueryQuotesInTableNameAndColumnName3() throws Exception {
+ String[] input = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = \"\"",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "select * from `test\"test` where `c'c` = \"\"",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testValidFreeFormQueryQuotesInTableNameAndColumnName4() throws Exception {
+ String[] input = new String[] {
+ "--query",
+ "select * from test where a = \"\\\"\"",
+ };
+
+ String[] output = new String[] {
+ "--query",
+ "select * from test where a = \"\\\"\"",
+ };
+
+ checkOptionsFile(input, output);
+ }
+
+ @Test
+ public void testInvalidFreeFormQueryEndingSingleQuoteOnly() throws Exception {
+ String[] input = new String[]{
+ "--query",
+ "SELECT * FROM table'",
+ };
+
+ checkInvalidOptionsFile(input);
+ }
+
+ @Test
+ public void testInvalidFreeFormQuerySingleQuoteStartDoubleQuoteEnd() throws Exception {
+
+ String[] input = new String[]{
+ "--query",
+ "'SELECT * FROM table\"",
+ };
+
+ checkInvalidOptionsFile(input);
+ }
+
private void checkInvalidOptionsFile(String[] fileContents) {
try {
checkOptionsFile(fileContents, new String[] {});