You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/07/11 17:35:12 UTC
git commit: SQOOP-1124: Direct export to PostgreSQL with
PGBulkloadManager fails without explicit port number settings
Updated Branches:
refs/heads/trunk 73cc549d9 -> ab4bfb9a8
SQOOP-1124: Direct export to PostgreSQL with PGBulkloadManager fails without explicit port number settings
(Masatake Iwasaki via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ab4bfb9a
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ab4bfb9a
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ab4bfb9a
Branch: refs/heads/trunk
Commit: ab4bfb9a8281d714b6549ba9d394d39e5c9c4d4a
Parents: 73cc549
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Jul 11 08:34:34 2013 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Jul 11 08:34:34 2013 -0700
----------------------------------------------------------------------
build.xml | 24 +++++++++++++++++++
.../postgresql/PGBulkloadExportJob.java | 1 -
.../postgresql/PGBulkloadExportMapper.java | 6 +++--
.../manager/PGBulkloadManagerManualTest.java | 25 +++++++++++++++-----
4 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index ef657e3..e3683f8 100644
--- a/build.xml
+++ b/build.xml
@@ -254,6 +254,18 @@
<property name="sqoop.test.postgresql.connectstring.host_url"
value="jdbc:postgresql://localhost/"/>
+ <property name="sqoop.test.postgresql.database"
+ value="sqooptest" />
+
+ <property name="sqoop.test.postgresql.tablespace"
+ value="sqooptest" />
+
+ <property name="sqoop.test.postgresql.username"
+ value="sqooptest" />
+
+ <property name="sqoop.test.postgresql.pg_bulkload"
+ value="pg_bulkload" />
+
<property name="sqoop.test.sqlserver.connectstring.host_url"
value="jdbc:sqlserver://sqlserverhost:1433"/>
@@ -818,6 +830,18 @@
<sysproperty key="sqoop.test.postgresql.connectstring.host_url"
value="${sqoop.test.postgresql.connectstring.host_url}"/>
+ <sysproperty key="sqoop.test.postgresql.database"
+ value="${sqoop.test.postgresql.database}" />
+
+ <sysproperty key="sqoop.test.postgresql.tablespace"
+ value="${sqoop.test.postgresql.tablespace}" />
+
+ <sysproperty key="sqoop.test.postgresql.username"
+ value="${sqoop.test.postgresql.username}" />
+
+ <sysproperty key="sqoop.test.postgresql.pg_bulkload"
+ value="${sqoop.test.postgresql.pg_bulkload}" />
+
<sysproperty key="sqoop.test.sqlserver.connectstring.host_url"
value="${sqoop.test.sqlserver.connectstring.host_url}"/>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
index 79fb7da..1e2ad9f 100644
--- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
@@ -148,7 +148,6 @@ public class PGBulkloadExportJob extends ExportJobBase {
@Override
public void runExport() throws ExportException, IOException {
- ConnManager cmgr = context.getConnManager();
SqoopOptions options = context.getOptions();
Configuration conf = options.getConf();
DBConfiguration dbConf = null;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
index 333546f..16af84c 100644
--- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
@@ -125,8 +125,10 @@ public class PGBulkloadExportMapper
+ JdbcUrl.getDatabaseName(conf.get(DBConfiguration.URL_PROPERTY)));
args.add("--host="
+ JdbcUrl.getHostName(conf.get(DBConfiguration.URL_PROPERTY)));
- args.add("--port="
- + JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY)));
+ int port = JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY));
+ if (port != -1) {
+ args.add("--port=" + port);
+ }
args.add("--input=stdin");
args.add("--output=" + tmpTableName);
args.add("-o");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
index 0403614..4d03a8b 100644
--- a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
import com.cloudera.sqoop.TestExport;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
@@ -64,10 +64,21 @@ public class PGBulkloadManagerManualTest extends TestExport {
public static final Log LOG =
LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
private DBConfiguration dbConf;
-
+ static final String HOST_URL =
+ System.getProperty("sqoop.test.postgresql.connectstring.host_url",
+ "jdbc:postgresql://localhost/");
+ static final String DATABASE =
+ System.getProperty("sqoop.test.postgresql.database", "sqooptest");
+ static final String TABLESPACE =
+ System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
+ static final String USERNAME =
+ System.getProperty("sqoop.test.postgresql.username", "sqooptest");
+ static final String PG_BULKLOAD =
+ System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
+ static final String CONNECT_STRING = HOST_URL + DATABASE;
public PGBulkloadManagerManualTest() {
- Configuration conf = getConf();
+ JobConf conf = new JobConf(getConf());
DBConfiguration.configureDB(conf,
"org.postgresql.Driver",
getConnectString(),
@@ -85,12 +96,12 @@ public class PGBulkloadManagerManualTest extends TestExport {
@Override
protected String getConnectString() {
- return "jdbc:postgresql://localhost:5432/sqooptest";
+ return CONNECT_STRING;
}
protected String getUserName() {
- return "sqooptest";
+ return USERNAME;
}
@@ -144,6 +155,8 @@ public class PGBulkloadManagerManualTest extends TestExport {
String... additionalArgv) {
ArrayList<String> args =
new ArrayList<String>(Arrays.asList(additionalArgv));
+ args.add("-D");
+ args.add("pgbulkload.bin=" + PG_BULKLOAD);
args.add("--username");
args.add(getUserName());
args.add("--connection-manager");
@@ -181,7 +194,7 @@ public class PGBulkloadManagerManualTest extends TestExport {
public void testExportWithTablespace() throws IOException, SQLException {
String[] genericargs =
- newStrArray(null, "-Dpgbulkload.staging.tablespace=sqooptest");
+ newStrArray(null, "-Dpgbulkload.staging.tablespace=" + TABLESPACE);
multiFileTestWithGenericArgs(1, 10, 1, genericargs);
}