You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by gw...@apache.org on 2015/04/07 00:37:49 UTC
[1/2] sqoop git commit: SQOOP-2282: Add validation check for
--hive-import and --append
Repository: sqoop
Updated Branches:
refs/heads/branch-1.4.6 2e75a215c -> 75c7b5ab4
SQOOP-2282: Add validation check for --hive-import and --append
(Abraham Elmahrek via Jarek Jarcec Cecho)
(cherry picked from commit 64c5fc3755bb26b48f515e1c2da6c5c5249e2b76)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9348d6a6
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9348d6a6
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9348d6a6
Branch: refs/heads/branch-1.4.6
Commit: 9348d6a63801e952117c3014cf71fdd773ec2c11
Parents: 2e75a21
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Apr 3 09:11:34 2015 -0700
Committer: Gwen Shapira <cs...@gmail.com>
Committed: Mon Apr 6 15:37:09 2015 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/tool/BaseSqoopTool.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9348d6a6/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 058e81d..c97bb58 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -1396,6 +1396,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
+ "importing into SequenceFile format.");
}
+ if (options.doHiveImport()
+ && options.isAppendMode()) {
+ throw new InvalidOptionsException("Append mode for hive imports is not "
+ + " yet supported. Please remove the parameter --append-mode");
+ }
+
// Many users are reporting issues when they are trying to import data
// directly into hive warehouse. This should prevent users from doing
// so in case of a default location.
[2/2] sqoop git commit: SQOOP-2281: Set overwrite on kite dataset
Posted by gw...@apache.org.
SQOOP-2281: Set overwrite on kite dataset
(Abraham Elmahrek via Jarek Jarcec Cecho)
(cherry picked from commit 032bc54c86c79f2f032211ce6505ff9c9fbce31e)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/75c7b5ab
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/75c7b5ab
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/75c7b5ab
Branch: refs/heads/branch-1.4.6
Commit: 75c7b5ab472649bf75bab1aac1db0155e691ff64
Parents: 9348d6a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Apr 3 09:45:55 2015 -0700
Committer: Gwen Shapira <cs...@gmail.com>
Committed: Mon Apr 6 15:37:25 2015 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/mapreduce/DataDrivenImportJob.java | 5 ++---
src/java/org/apache/sqoop/mapreduce/ParquetJob.java | 11 ++++++++---
src/test/com/cloudera/sqoop/hive/TestHiveImport.java | 3 +++
3 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
index bcf6611..e70d23c 100644
--- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
@@ -103,9 +103,8 @@ public class DataDrivenImportJob extends ImportJobBase {
// Parquet data records. The import will fail, if schema is invalid.
Schema schema = generateAvroSchema(tableName);
String uri = getKiteUri(conf, tableName);
- boolean reuseExistingDataset = options.isAppendMode() ||
- (options.doHiveImport() && options.doOverwriteHiveTable());
- ParquetJob.configureImportJob(conf, schema, uri, reuseExistingDataset);
+ ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(),
+ options.doHiveImport() && options.doOverwriteHiveTable());
}
job.setMapperClass(getMapperClass());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
index 85d88ff..df55dbc 100644
--- a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
@@ -71,9 +71,9 @@ public final class ParquetJob {
* {@link org.apache.avro.generic.GenericRecord}.
*/
public static void configureImportJob(Configuration conf, Schema schema,
- String uri, boolean reuseExistingDataset) throws IOException {
+ String uri, boolean reuseExistingDataset, boolean overwrite) throws IOException {
Dataset dataset;
- if (reuseExistingDataset) {
+ if (reuseExistingDataset || overwrite) {
try {
dataset = Datasets.load(uri);
} catch (DatasetNotFoundException ex) {
@@ -89,7 +89,12 @@ public final class ParquetJob {
dataset = createDataset(schema, getCompressionType(conf), uri);
}
conf.set(CONF_AVRO_SCHEMA, schema.toString());
- DatasetKeyOutputFormat.configure(conf).writeTo(dataset);
+
+ if (overwrite) {
+ DatasetKeyOutputFormat.configure(conf).overwrite(dataset);
+ } else {
+ DatasetKeyOutputFormat.configure(conf).writeTo(dataset);
+ }
}
private static Dataset createDataset(Schema schema,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
index d6df196..fa717cb 100644
--- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
+++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
@@ -307,6 +307,9 @@ public class TestHiveImport extends ImportJobTestCase {
runImportTest(TABLE_NAME, types, vals,
"createOverwriteImport.q", getCreateHiveTableArgs(extraArgs),
new CreateHiveTableTool());
+ runImportTest(TABLE_NAME, types, vals,
+ "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs),
+ new CreateHiveTableTool());
}
/** Test that dates are coerced properly to strings. */