You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by hs...@apache.org on 2015/09/24 23:41:27 UTC
sqoop git commit: SQOOP-2582. Query import won't work for parquet
Repository: sqoop
Updated Branches:
refs/heads/trunk fdcbaf5c5 -> ae9ab0fa3
SQOOP-2582. Query import won't work for parquet
(Jarcec via Hari)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ae9ab0fa
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ae9ab0fa
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ae9ab0fa
Branch: refs/heads/trunk
Commit: ae9ab0fa37efb405b349e2c91050479ef4355526
Parents: fdcbaf5
Author: Hari Shreedharan <hs...@apache.org>
Authored: Thu Sep 24 14:40:33 2015 -0700
Committer: Hari Shreedharan <hs...@apache.org>
Committed: Thu Sep 24 14:40:33 2015 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/tool/CodeGenTool.java | 2 +-
.../com/cloudera/sqoop/TestParquetImport.java | 43 ++++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ae9ab0fa/src/java/org/apache/sqoop/tool/CodeGenTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java
index 22ab030..b3107a2 100644
--- a/src/java/org/apache/sqoop/tool/CodeGenTool.java
+++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java
@@ -91,7 +91,7 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
}
LOG.info("Beginning code generation");
- if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
+ if (tableName != null && options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
String className = options.getClassName() != null ?
options.getClassName() : tableName;
if (className.equalsIgnoreCase(tableName)) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ae9ab0fa/src/test/com/cloudera/sqoop/TestParquetImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestParquetImport.java b/src/test/com/cloudera/sqoop/TestParquetImport.java
index ae2e617..43995c1 100644
--- a/src/test/com/cloudera/sqoop/TestParquetImport.java
+++ b/src/test/com/cloudera/sqoop/TestParquetImport.java
@@ -78,6 +78,31 @@ public class TestParquetImport extends ImportJobTestCase {
return args.toArray(new String[args.size()]);
}
+ protected String[] getOutputQueryArgv(boolean includeHadoopFlags, String[] extraArgs) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--query");
+ args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS");
+ args.add("--connect");
+ args.add(HsqldbTestServer.getUrl());
+ args.add("--target-dir");
+ args.add(getWarehouseDir() + "/" + getTableName());
+ args.add("--m");
+ args.add("1");
+ args.add("--split-by");
+ args.add("INTFIELD1");
+ args.add("--as-parquetfile");
+ if (extraArgs != null) {
+ args.addAll(Arrays.asList(extraArgs));
+ }
+
+ return args.toArray(new String[args.size()]);
+ }
+
public void testSnappyCompression() throws IOException {
runParquetImportTest("snappy");
}
@@ -198,6 +223,24 @@ public class TestParquetImport extends ImportJobTestCase {
}
}
+ public void testQueryImport() throws IOException, SQLException {
+ String [] types = { "INT" };
+ String [] vals = { "1" };
+ createTableWithColTypes(types, vals);
+
+ runImport(getOutputQueryArgv(true, null));
+
+ DatasetReader<GenericRecord> reader = getReader();
+ try {
+ assertTrue(reader.hasNext());
+ GenericRecord record1 = reader.next();
+ assertEquals(1, record1.get("DATA_COL0"));
+ assertFalse(reader.hasNext());
+ } finally {
+ reader.close();
+ }
+ }
+
public void testIncrementalParquetImport() throws IOException, SQLException {
String [] types = { "INT" };
String [] vals = { "1" };