You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ch...@apache.org on 2013/02/24 04:43:45 UTC
git commit: SQOOP-901: Allow user to override hardcoded boolean
strings in direct PostgreSQL connector
Updated Branches:
refs/heads/trunk 0affc4ba9 -> 8c3c78b11
SQOOP-901: Allow user to override hardcoded boolean strings in direct PostgreSQL connector
(Jarcec Cecho via Cheolsoo Park)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8c3c78b1
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8c3c78b1
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8c3c78b1
Branch: refs/heads/trunk
Commit: 8c3c78b11cb5c2330e31cbf867d32974a011f268
Parents: 0affc4b
Author: Cheolsoo Park <ch...@apache.org>
Authored: Sat Feb 23 19:31:59 2013 -0800
Committer: Cheolsoo Park <ch...@apache.org>
Committed: Sat Feb 23 19:31:59 2013 -0800
----------------------------------------------------------------------
src/docs/user/connectors.txt | 16 ++++
.../sqoop/manager/DirectPostgresqlManager.java | 69 ++++++++++++++-
.../apache/sqoop/manager/PostgresqlManager.java | 9 ++-
.../sqoop/manager/PostgresqlImportTest.java | 57 ++++++++----
4 files changed, 125 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/docs/user/connectors.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt
index ff424f7..7dd2a2e 100644
--- a/src/docs/user/connectors.txt
+++ b/src/docs/user/connectors.txt
@@ -98,6 +98,22 @@ Argument Description
Default is "public".
---------------------------------------------------------------------------------
+The direct connector (used when specified +\--direct+ parameter), offers also
+additional extra arguments:
+
+.Additional supported PostgreSQL extra arguments in direct mode:
+[grid="all"]
+`----------------------------------------`---------------------------------------
+Argument Description
+---------------------------------------------------------------------------------
++\--boolean-true-string <str>+ String that will be used to encode \
+ +true+ value of +boolean+ columns.
+ Default is "TRUE".
++\--boolean-false-string <str>+ String that will be used to encode \
+ +false+ value of +boolean+ columns.
+ Default is "FALSE".
+---------------------------------------------------------------------------------
+
Schema support
^^^^^^^^^^^^^^
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
index a05bf60..c085218 100644
--- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
@@ -30,9 +30,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sqoop.cli.RelatedOptions;
import org.apache.sqoop.util.PostgreSQLUtils;
import com.cloudera.sqoop.SqoopOptions;
@@ -54,15 +57,33 @@ import org.apache.sqoop.util.SubstitutionUtils;
*/
public class DirectPostgresqlManager
extends com.cloudera.sqoop.manager.PostgresqlManager {
+
public static final Log LOG = LogFactory.getLog(
DirectPostgresqlManager.class.getName());
+ public static final String BOOLEAN_TRUE_STRING = "boolean-true-string";
+ public static final String DEFAULT_BOOLEAN_TRUE_STRING = "TRUE";
+
+ public static final String BOOLEAN_FALSE_STRING = "boolean-false-string";
+ public static final String DEFAULT_BOOLEAN_FALSE_STRING = "FALSE";
+
public DirectPostgresqlManager(final SqoopOptions opts) {
super(opts);
+
+ if (this.booleanFalseString == null) {
+ this.booleanFalseString = DEFAULT_BOOLEAN_FALSE_STRING;
+ }
+ if (booleanTrueString == null) {
+ this.booleanTrueString = DEFAULT_BOOLEAN_TRUE_STRING;
+ }
}
private static final String PSQL_CMD = "psql";
+ private String booleanTrueString;
+
+ private String booleanFalseString;
+
/** Copies data directly into HDFS, adding the user's chosen line terminator
char to each record.
*/
@@ -196,12 +217,18 @@ public class DirectPostgresqlManager
sb.append(col);
} else {
if ("bool".equalsIgnoreCase(columnTypes.get(col))) {
- sb.append(String.format("case when %s=true then 'TRUE' "
- + "when %s=false then 'FALSE' end as %s",
+ sb.append(String.format("case when %s=true then "
+ + "'" + booleanTrueString + "' "
+ + "when %s=false then "
+ + "'" + booleanFalseString + "'"
+ + " end as %s",
colEscaped, colEscaped, colEscaped));
} else if ("bit".equalsIgnoreCase(columnTypes.get(col))) {
- sb.append(String.format("case when %s=B'1' then 'TRUE' "
- + "when %s=B'0' then 'FALSE' end as %s",
+ sb.append(String.format("case when %s=B'1' then "
+ + "'" + booleanTrueString + "' "
+ + "when %s=B'0' then "
+ + "'" + booleanFalseString + "'"
+ + " end as %s",
colEscaped, colEscaped, colEscaped));
} else {
sb.append(colEscaped);
@@ -508,4 +535,38 @@ public class DirectPostgresqlManager
return false;
}
// CHECKSTYLE:ON
+
+ /** {@inheritDoc}. */
+ @Override
+ protected void applyExtraArguments(CommandLine cmdLine) {
+ super.applyExtraArguments(cmdLine);
+
+ if (cmdLine.hasOption(BOOLEAN_TRUE_STRING)) {
+ String arg = cmdLine.getOptionValue(BOOLEAN_TRUE_STRING);
+ LOG.info("Loaded TRUE encoding string " + arg);
+ this.booleanTrueString = arg;
+ }
+ if (cmdLine.hasOption(BOOLEAN_FALSE_STRING)) {
+ String arg = cmdLine.getOptionValue(BOOLEAN_FALSE_STRING);
+ LOG.info("Loaded FALSE encoding string " + arg);
+ this.booleanFalseString = arg;
+ }
+ }
+
+ /** {@inheritDoc}. */
+ @Override
+ @SuppressWarnings("static-access")
+ protected RelatedOptions getExtraOptions() {
+ RelatedOptions extraOptions = super.getExtraOptions();
+
+ extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
+ .withDescription("String to encode TRUE value")
+ .withLongOpt(BOOLEAN_TRUE_STRING).create());
+
+ extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
+ .withDescription("String to encode FALSE value")
+ .withLongOpt(BOOLEAN_FALSE_STRING).create());
+
+ return extraOptions;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/PostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/PostgresqlManager.java b/src/java/org/apache/sqoop/manager/PostgresqlManager.java
index facdc74..609a457 100644
--- a/src/java/org/apache/sqoop/manager/PostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/PostgresqlManager.java
@@ -196,7 +196,7 @@ public class PostgresqlManager
* @param args Extra arguments array
* @throws ParseException
*/
- void parseExtraArgs(String[] args) throws ParseException {
+ private void parseExtraArgs(String[] args) throws ParseException {
// No-op when no extra arguments are present
if (args == null || args.length == 0) {
return;
@@ -207,6 +207,11 @@ public class PostgresqlManager
CommandLineParser parser = new GnuParser();
CommandLine cmdLine = parser.parse(getExtraOptions(), args, true);
+ // Apply parsed arguments
+ applyExtraArguments(cmdLine);
+ }
+
+ protected void applyExtraArguments(CommandLine cmdLine) {
// Apply extra options
if (cmdLine.hasOption(SCHEMA)) {
String schemaName = cmdLine.getOptionValue(SCHEMA);
@@ -222,7 +227,7 @@ public class PostgresqlManager
* @return
*/
@SuppressWarnings("static-access")
- private RelatedOptions getExtraOptions() {
+ protected RelatedOptions getExtraOptions() {
// Connection args (common)
RelatedOptions extraOptions =
new RelatedOptions("PostgreSQL extra options:");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
index ee00c41..3fadff7 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
@@ -139,12 +139,13 @@ public class PostgresqlImportTest extends ImportJobTestCase {
connection.commit();
} catch (SQLException e) {
LOG.info("Couldn't create schema " + schema + " (is o.k. as long as"
- + "the schema already exists.", e);
+ + "the schema already exists.");
connection.rollback();
}
String fullTableName = manager.escapeTableName(schema)
+ "." + manager.escapeTableName(tableName);
+ LOG.info("Creating table: " + fullTableName);
try {
// Try to remove the table first. DROP TABLE IF EXISTS didn't
@@ -152,8 +153,7 @@ public class PostgresqlImportTest extends ImportJobTestCase {
// any exception here if one occurs.
st.executeUpdate("DROP TABLE " + fullTableName);
} catch (SQLException e) {
- LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)",
- e);
+ LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)");
// Now we need to reset the transaction.
connection.rollback();
}
@@ -163,17 +163,18 @@ public class PostgresqlImportTest extends ImportJobTestCase {
+ manager.escapeColName("name") + " VARCHAR(24) NOT NULL, "
+ manager.escapeColName("start_date") + " DATE, "
+ manager.escapeColName("Salary") + " FLOAT, "
+ + manager.escapeColName("Fired") + " BOOL, "
+ manager.escapeColName("dept") + " VARCHAR(32))");
st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(1,'Aaron','2009-05-14',1000000.00,'engineering')");
+ + " VALUES(1,'Aaron','2009-05-14',1000000.00,TRUE,'engineering')");
st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(2,'Bob','2009-04-20',400.00,'sales')");
+ + " VALUES(2,'Bob','2009-04-20',400.00,TRUE,'sales')");
st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(3,'Fred','2009-01-23',15.00,'marketing')");
+ + " VALUES(3,'Fred','2009-01-23',15.00,FALSE,'marketing')");
if (nullEntry) {
st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(4,'Mike',NULL,NULL,NULL)");
+ + " VALUES(4,'Mike',NULL,NULL,NULL,NULL)");
}
connection.commit();
@@ -276,8 +277,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testJdbcBasedImport() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400.0,sales",
- "3,Fred,2009-01-23,15.0,marketing",
+ "2,Bob,2009-04-20,400.0,true,sales",
+ "3,Fred,2009-01-23,15.0,false,marketing",
};
doImportAndVerify(false, expectedResults, TABLE_NAME);
@@ -286,8 +287,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testDirectImport() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
+ "2,Bob,2009-04-20,400,TRUE,sales",
+ "3,Fred,2009-01-23,15,FALSE,marketing",
};
doImportAndVerify(true, expectedResults, TABLE_NAME);
@@ -309,8 +310,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testTableNameWithSpecialCharacter() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400.0,sales",
- "3,Fred,2009-01-23,15.0,marketing",
+ "2,Bob,2009-04-20,400.0,true,sales",
+ "3,Fred,2009-01-23,15.0,false,marketing",
};
doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME);
@@ -330,8 +331,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testDifferentSchemaImport() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400.0,sales",
- "3,Fred,2009-01-23,15.0,marketing",
+ "2,Bob,2009-04-20,400.0,true,sales",
+ "3,Fred,2009-01-23,15.0,false,marketing",
};
String [] extraArgs = { "--",
@@ -344,8 +345,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testDifferentSchemaImportDirect() throws IOException {
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
+ "2,Bob,2009-04-20,400,TRUE,sales",
+ "3,Fred,2009-01-23,15,FALSE,marketing",
};
String [] extraArgs = { "--",
@@ -358,9 +359,9 @@ public class PostgresqlImportTest extends ImportJobTestCase {
@Test
public void testNullEscapeCharacters() throws Exception {
String [] expectedResults = {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- "4,Mike,\\N,\\N,\\N",
+ "2,Bob,2009-04-20,400,TRUE,sales",
+ "3,Fred,2009-01-23,15,FALSE,marketing",
+ "4,Mike,\\N,\\N,\\N,\\N",
};
String [] extraArgs = {
@@ -370,4 +371,20 @@ public class PostgresqlImportTest extends ImportJobTestCase {
doImportAndVerify(true, expectedResults, NULL_TABLE_NAME, extraArgs);
}
+
+ @Test
+ public void testDifferentBooleanValues() throws Exception {
+ String [] expectedResults = {
+ "2,Bob,2009-04-20,400,REAL_TRUE,sales",
+ "3,Fred,2009-01-23,15,REAL_FALSE,marketing",
+ };
+
+ String [] extraArgs = {
+ "--",
+ "--boolean-true-string", "REAL_TRUE",
+ "--boolean-false-string", "REAL_FALSE",
+ };
+
+ doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+ }
}