You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sqoop.apache.org by Jarek Cecho <ja...@apache.org> on 2015/07/22 23:32:42 UTC
Re: Review Request 31561: Sqoop2: Hdfs Intermediate Data Format
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/31561/#review92656
-----------------------------------------------------------
Thank you for taking a stab at this one Abe. I have one high level comment:
I would prefer to rip the changes to IntermedidateDataFormat out of this patch and have them covered by standalone JIRA (e.g. provide configuration options to IDF). It seems quite important change to our infrastructure so it would be great to have it separately.
Also do you think it would be worth it to provide integration tests covering end-to-end use case for both "import" and "export"?
Jarcec
- Jarek Cecho
On June 16, 2015, 4:11 p.m., Abraham Elmahrek wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/31561/
> -----------------------------------------------------------
>
> (Updated June 16, 2015, 4:11 p.m.)
>
>
> Review request for Sqoop.
>
>
> Bugs: SQOOP-2133
> https://issues.apache.org/jira/browse/SQOOP-2133
>
>
> Repository: sqoop-sqoop2
>
>
> Description
> -------
>
> commit f0c7c5092ac7d376ced8998301395399a429a0ba
> Author: Abraham Elmahrek <ab...@apache.org>
> Date: Thu Feb 26 13:34:34 2015 -0800
>
> SQOOP-2133: Sqoop2: Hdfs Intermediate Data Format
>
> :100644 100644 2d5ba58... a42f214... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
> :100644 100644 8237e51... 6483f35... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
> :000000 100644 0000000... 37e3261... A connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java
> :100644 100644 cee0a91... 42f6f8a... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
> :100644 100644 fce7728... ed6ec58... M connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java
> :100644 100644 9fcd2a8... 6ed944d... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
> :000000 100644 0000000... b2c00d8... A connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java
> :100644 100644 c560c08... d84631c... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java
> :100644 100644 b7c81ec... f7a3a49... M connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
> :100644 100644 c460f80... 546806e... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java
> :100644 100644 fef8202... cf88ca7... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java
> :100644 100644 76eaa67... 52fa628... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
> :100644 100644 f4cd044... 317dd61... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
> :100644 100644 0fac41a... c48bb24... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java
> :100644 100644 bfd5f23... a6edc70... M connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
> :100644 100644 dc441bc... 376d262... M core/src/main/java/org/apache/sqoop/driver/JobManager.java
> :100644 100644 cfa45b2... 58dd6e8... M core/src/main/java/org/apache/sqoop/driver/JobRequest.java
> :100644 100644 59ad311... 2ececd0... M execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java
> :100644 100644 d6fe6af... eae2886... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
> :100644 100644 fc18586... 4f8cb1d... M execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
> :100644 100644 10ed33c... 193148e... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
> :100644 100644 f658d09... a09eb67... M execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
> :100644 100644 85bf73d... b354e43... M execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java
> :100644 100644 71c98db... 4c6bf8d... M execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
> :100644 100644 92ad1b8... e2eb508... M execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java
>
>
> Diffs
> -----
>
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java 2d5ba58
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b35c957
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsIntermediateDataFormat.java PRE-CREATION
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java 96913e8
> connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsUtils.java 46e09ab
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java 03b13bd
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsIntermediateDataFormat.java PRE-CREATION
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsUtils.java c560c08
> connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java 688067b
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java 2a7aa1b
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java fef8202
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java 76eaa67
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java f4cd044
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java 0fac41a
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java bfd5f23
> core/src/main/java/org/apache/sqoop/driver/JobManager.java e8ca17c
> core/src/main/java/org/apache/sqoop/driver/JobRequest.java cfa45b2
> execution/mapreduce/src/main/java/org/apache/sqoop/job/io/SqoopWritable.java 59ad311
> execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java 14fdfdc
> execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java c9d6f10
> execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java 10ed33c
> execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java f658d09
> execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestSqoopWritable.java 85bf73d
> execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java 3208e8a
> execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java 92ad1b8
>
> Diff: https://reviews.apache.org/r/31561/diff/
>
>
> Testing
> -------
>
> mvn clean test
> Manually tested null override in HDFS connector
>
>
> Thanks,
>
> Abraham Elmahrek
>
>