You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2015/05/30 03:28:57 UTC
sqoop git commit: SQOOP-2372: Import all tables as parquet will throw
NPE
Repository: sqoop
Updated Branches:
refs/heads/trunk 974b886c4 -> 9147967ee
SQOOP-2372: Import all tables as parquet will throw NPE
(Qian Xu via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9147967e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9147967e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9147967e
Branch: refs/heads/trunk
Commit: 9147967eee4c72997fa858e2bbc8b9b7a58b994c
Parents: 974b886
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Fri May 29 17:31:44 2015 -0700
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Fri May 29 18:28:15 2015 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/tool/CodeGenTool.java | 4 +-
src/test/com/cloudera/sqoop/TestAllTables.java | 49 +++++++++++++++++---
2 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9147967e/src/java/org/apache/sqoop/tool/CodeGenTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java
index ab339ad..22ab030 100644
--- a/src/java/org/apache/sqoop/tool/CodeGenTool.java
+++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java
@@ -93,8 +93,8 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
String className = options.getClassName() != null ?
- options.getClassName() : options.getTableName();
- if (className.equalsIgnoreCase(options.getTableName())) {
+ options.getClassName() : tableName;
+ if (className.equalsIgnoreCase(tableName)) {
className = "codegen_" + className;
options.setClassName(className);
LOG.info("Will generate java class as " + options.getClassName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9147967e/src/test/com/cloudera/sqoop/TestAllTables.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAllTables.java b/src/test/com/cloudera/sqoop/TestAllTables.java
index 39e8941..f981024 100644
--- a/src/test/com/cloudera/sqoop/TestAllTables.java
+++ b/src/test/com/cloudera/sqoop/TestAllTables.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.avro.generic.GenericRecord;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +35,9 @@ import org.junit.After;
import com.cloudera.sqoop.testutil.CommonArgs;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportAllTablesTool;
+import org.kitesdk.data.Dataset;
+import org.kitesdk.data.DatasetReader;
+import org.kitesdk.data.Datasets;
/**
* Test the --all-tables functionality that can import multiple tables.
@@ -44,13 +48,10 @@ public class TestAllTables extends ImportJobTestCase {
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
- private String [] getArgv(boolean includeHadoopFlags, String[] excludeTables) {
+ private String [] getArgv(String[] extraArgs, String[] excludeTables) {
ArrayList<String> args = new ArrayList<String>();
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
+ CommonArgs.addHadoopFlags(args);
args.add("--warehouse-dir");
args.add(getWarehouseDir());
args.add("--connect");
@@ -63,6 +64,11 @@ public class TestAllTables extends ImportJobTestCase {
args.add("--exclude-tables");
args.add(StringUtils.join(excludeTables, ","));
}
+ if (extraArgs != null) {
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
+ }
return args.toArray(new String[0]);
}
@@ -124,7 +130,7 @@ public class TestAllTables extends ImportJobTestCase {
}
public void testMultiTableImport() throws IOException {
- String [] argv = getArgv(true, null);
+ String [] argv = getArgv(null, null);
runImport(new ImportAllTablesTool(), argv);
Path warehousePath = new Path(this.getWarehouseDir());
@@ -159,9 +165,38 @@ public class TestAllTables extends ImportJobTestCase {
}
}
+ public void testMultiTableImportAsParquetFormat() throws IOException {
+ String [] argv = getArgv(new String[]{"--as-parquetfile"}, null);
+ runImport(new ImportAllTablesTool(), argv);
+
+ Path warehousePath = new Path(this.getWarehouseDir());
+ int i = 0;
+ for (String tableName : this.tableNames) {
+ Path tablePath = new Path(warehousePath, tableName);
+ Dataset dataset = Datasets.load("dataset:file:" + tablePath);
+
+ // dequeue the expected value for this table. This
+ // list has the same order as the tableNames list.
+ String expectedVal = Integer.toString(i++) + ","
+ + this.expectedStrings.get(0);
+ this.expectedStrings.remove(0);
+
+ DatasetReader<GenericRecord> reader = dataset.newReader();
+ try {
+ GenericRecord record = reader.next();
+ String line = record.get(0) + "," + record.get(1);
+ assertEquals("Table " + tableName + " expected a different string",
+ expectedVal, line);
+ assertFalse(reader.hasNext());
+ } finally {
+ reader.close();
+ }
+ }
+ }
+
public void testMultiTableImportWithExclude() throws IOException {
String exclude = this.tableNames.get(0);
- String [] argv = getArgv(true, new String[]{ exclude });
+ String [] argv = getArgv(null, new String[]{ exclude });
runImport(new ImportAllTablesTool(), argv);
Path warehousePath = new Path(this.getWarehouseDir());