You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/03 18:47:07 UTC
sqoop git commit: SQOOP-2281: Set overwrite on kite dataset
Repository: sqoop
Updated Branches:
refs/heads/trunk 64c5fc375 -> 032bc54c8
SQOOP-2281: Set overwrite on kite dataset
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/032bc54c
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/032bc54c
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/032bc54c
Branch: refs/heads/trunk
Commit: 032bc54c86c79f2f032211ce6505ff9c9fbce31e
Parents: 64c5fc3
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Apr 3 09:45:55 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Fri Apr 3 09:45:55 2015 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/mapreduce/DataDrivenImportJob.java | 5 ++---
src/java/org/apache/sqoop/mapreduce/ParquetJob.java | 11 ++++++++---
src/test/com/cloudera/sqoop/hive/TestHiveImport.java | 3 +++
3 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
index bcf6611..e70d23c 100644
--- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
@@ -103,9 +103,8 @@ public class DataDrivenImportJob extends ImportJobBase {
// Parquet data records. The import will fail, if schema is invalid.
Schema schema = generateAvroSchema(tableName);
String uri = getKiteUri(conf, tableName);
- boolean reuseExistingDataset = options.isAppendMode() ||
- (options.doHiveImport() && options.doOverwriteHiveTable());
- ParquetJob.configureImportJob(conf, schema, uri, reuseExistingDataset);
+ ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(),
+ options.doHiveImport() && options.doOverwriteHiveTable());
}
job.setMapperClass(getMapperClass());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
index 85d88ff..df55dbc 100644
--- a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java
@@ -71,9 +71,9 @@ public final class ParquetJob {
* {@link org.apache.avro.generic.GenericRecord}.
*/
public static void configureImportJob(Configuration conf, Schema schema,
- String uri, boolean reuseExistingDataset) throws IOException {
+ String uri, boolean reuseExistingDataset, boolean overwrite) throws IOException {
Dataset dataset;
- if (reuseExistingDataset) {
+ if (reuseExistingDataset || overwrite) {
try {
dataset = Datasets.load(uri);
} catch (DatasetNotFoundException ex) {
@@ -89,7 +89,12 @@ public final class ParquetJob {
dataset = createDataset(schema, getCompressionType(conf), uri);
}
conf.set(CONF_AVRO_SCHEMA, schema.toString());
- DatasetKeyOutputFormat.configure(conf).writeTo(dataset);
+
+ if (overwrite) {
+ DatasetKeyOutputFormat.configure(conf).overwrite(dataset);
+ } else {
+ DatasetKeyOutputFormat.configure(conf).writeTo(dataset);
+ }
}
private static Dataset createDataset(Schema schema,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
index d6df196..fa717cb 100644
--- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
+++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
@@ -307,6 +307,9 @@ public class TestHiveImport extends ImportJobTestCase {
runImportTest(TABLE_NAME, types, vals,
"createOverwriteImport.q", getCreateHiveTableArgs(extraArgs),
new CreateHiveTableTool());
+ runImportTest(TABLE_NAME, types, vals,
+ "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs),
+ new CreateHiveTableTool());
}
/** Test that dates are coerced properly to strings. */