You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/01/06 10:15:11 UTC
sqoop git commit: SQOOP-1826: NPE in ImportTool.lastModifiedMerge
during postgres import
Repository: sqoop
Updated Branches:
refs/heads/trunk 8b6eb33f1 -> c32010f44
SQOOP-1826: NPE in ImportTool.lastModifiedMerge during postgres import
(Ricky Nguyen via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c32010f4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c32010f4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c32010f4
Branch: refs/heads/trunk
Commit: c32010f440f139b37840df6a3433f875898136d5
Parents: 8b6eb33
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Jan 6 10:14:38 2015 +0100
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Jan 6 10:14:38 2015 +0100
----------------------------------------------------------------------
.../sqoop/manager/DirectPostgresqlManager.java | 2 ++
.../sqoop/manager/PostgresqlImportTest.java | 37 ++++++++++++++++++--
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
index 8d4a097..63b0704 100644
--- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
@@ -348,6 +348,8 @@ public class DirectPostgresqlManager
public void importTable(com.cloudera.sqoop.manager.ImportJobContext context)
throws IOException, ImportException {
+ context.setConnManager(this);
+
String tableName = context.getTableName();
SqoopOptions options = context.getOptions();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
index 3b5731b..57dd338 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
@@ -235,15 +235,18 @@ public class PostgresqlImportTest extends ImportJobTestCase {
return args.toArray(new String[0]);
}
- private void doImportAndVerify(boolean isDirect, String [] expectedResults,
+ private void doImportAndVerify(boolean isDirect, String[] expectedResults,
String tableName, String... extraArgs) throws IOException {
Path warehousePath = new Path(this.getWarehouseDir());
Path tablePath = new Path(warehousePath, tableName);
- Path filePath = new Path(tablePath, "part-m-00000");
+
+ // if importing with merge step, directory should exist and output should be from a reducer
+ boolean isMerge = Arrays.asList(extraArgs).contains("--merge-key");
+ Path filePath = new Path(tablePath, isMerge ? "part-r-00000" : "part-m-00000");
File tableFile = new File(tablePath.toString());
- if (tableFile.exists() && tableFile.isDirectory()) {
+ if (tableFile.exists() && tableFile.isDirectory() && !isMerge) {
// remove the directory before running the import.
FileListing.recursiveDeleteDir(tableFile);
}
@@ -329,6 +332,34 @@ public class PostgresqlImportTest extends ImportJobTestCase {
doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs);
}
+ public void testDirectIncrementalImport() throws IOException {
+ String [] expectedResults = { };
+
+ String [] extraArgs = { "--incremental", "lastmodified",
+ "--check-column", "start_date",
+ };
+
+ doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+ }
+
+ public void testDirectIncrementalImportMerge() throws IOException {
+ String [] expectedResults = { };
+
+ String [] extraArgs = { "--incremental", "lastmodified",
+ "--check-column", "start_date",
+ };
+
+ doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+
+ extraArgs = new String[] { "--incremental", "lastmodified",
+ "--check-column", "start_date",
+ "--merge-key", "id",
+ "--last-value", "2009-04-20"
+ };
+
+ doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+ }
+
@Test
public void testDifferentSchemaImport() throws IOException {
String [] expectedResults = {