You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sqoop.apache.org by Abraham Elmahrek <ab...@cloudera.com> on 2015/06/16 18:11:07 UTC
Re: Review Request 31561: Sqoop2: Hdfs Intermediate Data Format
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/31561/
-----------------------------------------------------------
(Updated June 16, 2015, 4:11 p.m.)
Review request for Sqoop.
Bugs: SQOOP-2133
https://issues.apache.org/jira/browse/SQOOP-2133
Repository: sqoop-sqoop2
Description
-------
commit f0c7c5092ac7d376ced8998301395399a429a0ba
Author: Abraham Elmahrek <ab...@apache.org>
Date: Thu Feb 26 13:34:34 2015 -0800
SQOOP-2133: Sqoop2: Hdfs Intermediate Data Format
:100644 100644 2d5ba58... a42f214... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
:100644 100644 8237e51... 6483f35... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
:000000 100644 0000000... 37e3261... A connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java
:100644 100644 cee0a91... 42f6f8a... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
:100644 100644 fce7728... ed6ec58... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java
:100644 100644 9fcd2a8... 6ed944d... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
:000000 100644 0000000... b2c00d8... A connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java
:100644 100644 c560c08... d84631c... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java
:100644 100644 b7c81ec... f7a3a49... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
:100644 100644 c460f80... 546806e... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java
:100644 100644 fef8202... cf88ca7... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
:100644 100644 76eaa67... 52fa628... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
:100644 100644 f4cd044... 317dd61... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
:100644 100644 0fac41a... c48bb24... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java
:100644 100644 bfd5f23... a6edc70... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
:100644 100644 dc441bc... 376d262... M core/src/main/java/org/apache/sqoop/driver/JobManager.java
:100644 100644 cfa45b2... 58dd6e8... M core/src/main/java/org/apache/sqoop/driver/JobRequest.java
:100644 100644 59ad311... 2ececd0... M execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
:100644 100644 d6fe6af... eae2886... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
:100644 100644 fc18586... 4f8cb1d... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
:100644 100644 10ed33c... 193148e... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
:100644 100644 f658d09... a09eb67... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
:100644 100644 85bf73d... b354e43... M execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
:100644 100644 71c98db... 4c6bf8d... M execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
:100644 100644 92ad1b8... e2eb508... M execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
Diffs (updated)
-----
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java 2d5ba58
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b35c957
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java PRE-CREATION
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java 96913e8
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java 46e09ab
connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java 03b13bd
connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java PRE-CREATION
connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java c560c08
connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java 688067b
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java 2a7aa1b
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java fef8202
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java 76eaa67
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java f4cd044
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java 0fac41a
connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java bfd5f23
core/src/main/java/org/apache/sqoop/driver/JobManager.java e8ca17c
core/src/main/java/org/apache/sqoop/driver/JobRequest.java cfa45b2
execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java 59ad311
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java 14fdfdc
execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java c9d6f10
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java 10ed33c
execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java f658d09
execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java 85bf73d
execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java 3208e8a
execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java 92ad1b8
Diff: https://reviews.apache.org/r/31561/diff/
Testing
-------
mvn clean test
Manually tested null override in HDFS connector
Thanks,
Abraham Elmahrek
Re: Review Request 31561: Sqoop2: Hdfs Intermediate Data Format
Posted by Jarek Cecho <ja...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/31561/#review92656
-----------------------------------------------------------
Thank you for taking a stab at this one Abe. I have one high level comment:
I would prefer to rip the changes to IntermedidateDataFormat out of this patch and have them covered by standalone JIRA (e.g. provide configuration options to IDF). It seems quite important change to our infrastructure so it would be great to have it separately.
Also do you think it would be worth it to provide integration tests covering end-to-end use case for both "import" and "export"?
Jarcec
- Jarek Cecho
On June 16, 2015, 4:11 p.m., Abraham Elmahrek wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/31561/
> -----------------------------------------------------------
>
> (Updated June 16, 2015, 4:11 p.m.)
>
>
> Review request for Sqoop.
>
>
> Bugs: SQOOP-2133
> https://issues.apache.org/jira/browse/SQOOP-2133
>
>
> Repository: sqoop-sqoop2
>
>
> Description
> -------
>
> commit f0c7c5092ac7d376ced8998301395399a429a0ba
> Author: Abraham Elmahrek <ab...@apache.org>
> Date: Thu Feb 26 13:34:34 2015 -0800
>
> SQOOP-2133: Sqoop2: Hdfs Intermediate Data Format
>
> :100644 100644 2d5ba58... a42f214... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
> :100644 100644 8237e51... 6483f35... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
> :000000 100644 0000000... 37e3261... A connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java
> :100644 100644 cee0a91... 42f6f8a... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
> :100644 100644 fce7728... ed6ec58... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java
> :100644 100644 9fcd2a8... 6ed944d... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
> :000000 100644 0000000... b2c00d8... A connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java
> :100644 100644 c560c08... d84631c... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java
> :100644 100644 b7c81ec... f7a3a49... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
> :100644 100644 c460f80... 546806e... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java
> :100644 100644 fef8202... cf88ca7... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
> :100644 100644 76eaa67... 52fa628... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
> :100644 100644 f4cd044... 317dd61... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
> :100644 100644 0fac41a... c48bb24... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java
> :100644 100644 bfd5f23... a6edc70... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
> :100644 100644 dc441bc... 376d262... M core/src/main/java/org/apache/sqoop/driver/JobManager.java
> :100644 100644 cfa45b2... 58dd6e8... M core/src/main/java/org/apache/sqoop/driver/JobRequest.java
> :100644 100644 59ad311... 2ececd0... M execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
> :100644 100644 d6fe6af... eae2886... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
> :100644 100644 fc18586... 4f8cb1d... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
> :100644 100644 10ed33c... 193148e... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
> :100644 100644 f658d09... a09eb67... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
> :100644 100644 85bf73d... b354e43... M execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
> :100644 100644 71c98db... 4c6bf8d... M execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
> :100644 100644 92ad1b8... e2eb508... M execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
>
>
> Diffs
> -----
>
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java 2d5ba58
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b35c957
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java PRE-CREATION
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java 96913e8
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java 46e09ab
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java 03b13bd
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java PRE-CREATION
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java c560c08
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java 688067b
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java 2a7aa1b
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java fef8202
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java 76eaa67
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java f4cd044
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java 0fac41a
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java bfd5f23
> core/src/main/java/org/apache/sqoop/driver/JobManager.java e8ca17c
> core/src/main/java/org/apache/sqoop/driver/JobRequest.java cfa45b2
> execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java 59ad311
> execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java 14fdfdc
> execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java c9d6f10
> execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java 10ed33c
> execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java f658d09
> execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java 85bf73d
> execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java 3208e8a
> execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java 92ad1b8
>
> Diff: https://reviews.apache.org/r/31561/diff/
>
>
> Testing
> -------
>
> mvn clean test
> Manually tested null override in HDFS connector
>
>
> Thanks,
>
> Abraham Elmahrek
>
>