You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hive.apache.org by Ashutosh Chauhan <ha...@apache.org> on 2013/12/31 01:42:11 UTC
Review Request 16526: more usage of paths
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/16526/
-----------------------------------------------------------
Review request for hive, Xuefu Zhang and Xuefu Zhang.
Bugs: HIVE-6121
https://issues.apache.org/jira/browse/HIVE-6121
Repository: hive
Description
-------
Refactoring patch.
Diffs
-----
trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadMultiFilesDesc.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 1554326
trunk/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java 1554326
trunk/ql/src/test/results/clientpositive/binary_output_format.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucket1.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucket2.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucket3.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucket4.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucket5.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out 1554326
trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out 1554326
trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out 1554326
trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out 1554326
trunk/ql/src/test/results/clientpositive/input_part1.q.out 1554326
trunk/ql/src/test/results/clientpositive/input_part2.q.out 1554326
trunk/ql/src/test/results/clientpositive/join17.q.out 1554326
trunk/ql/src/test/results/clientpositive/join26.q.out 1554326
trunk/ql/src/test/results/clientpositive/join32.q.out 1554326
trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out 1554326
trunk/ql/src/test/results/clientpositive/join33.q.out 1554326
trunk/ql/src/test/results/clientpositive/join34.q.out 1554326
trunk/ql/src/test/results/clientpositive/join35.q.out 1554326
trunk/ql/src/test/results/clientpositive/join9.q.out 1554326
trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out 1554326
trunk/ql/src/test/results/clientpositive/list_bucket_dml_10.q.out 1554326
trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out 1554326
trunk/ql/src/test/results/clientpositive/merge3.q.out 1554326
trunk/ql/src/test/results/clientpositive/pcr.q.out 1554326
trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out 1554326
trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample1.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample2.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample4.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample5.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample6.q.out 1554326
trunk/ql/src/test/results/clientpositive/sample7.q.out 1554326
trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out 1554326
trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out 1554326
trunk/ql/src/test/results/clientpositive/stats0.q.out 1554326
trunk/ql/src/test/results/clientpositive/stats11.q.out 1554326
trunk/ql/src/test/results/clientpositive/stats3.q.out 1554326
trunk/ql/src/test/results/clientpositive/union22.q.out 1554326
trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/groupby1.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input1.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input2.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input3.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input4.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input5.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input6.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input7.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input9.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/join1.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/join2.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/join3.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample2.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample3.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample4.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample5.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample6.q.xml 1554326
trunk/ql/src/test/results/compiler/plan/sample7.q.xml 1554326
Diff: https://reviews.apache.org/r/16526/diff/
Testing
-------
No new functionality. Existing regression suite suffices.
Thanks,
Ashutosh Chauhan
Re: Review Request 16526: more usage of paths
Posted by Ashutosh Chauhan <ha...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/16526/
-----------------------------------------------------------
(Updated Dec. 31, 2013, 5:23 p.m.)
Review request for hive, Xuefu Zhang and Xuefu Zhang.
Changes
-------
* Rebased after HIVE-6116 went in trunk.
* Whitespace changes for indentation.
* Updated .q.out for failed tests.
* No other code changes.
Bugs: HIVE-6121
https://issues.apache.org/jira/browse/HIVE-6121
Repository: hive
Description
-------
Refactoring patch.
Diffs (updated)
-----
trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554560
trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadMultiFilesDesc.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java 1554560
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 1554560
trunk/ql/src/test/results/clientpositive/binary_output_format.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucket1.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucket2.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucket3.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucket4.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucket5.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out 1554560
trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out 1554560
trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out 1554560
trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out 1554560
trunk/ql/src/test/results/clientpositive/input_part1.q.out 1554560
trunk/ql/src/test/results/clientpositive/input_part2.q.out 1554560
trunk/ql/src/test/results/clientpositive/join17.q.out 1554560
trunk/ql/src/test/results/clientpositive/join26.q.out 1554560
trunk/ql/src/test/results/clientpositive/join32.q.out 1554560
trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out 1554560
trunk/ql/src/test/results/clientpositive/join33.q.out 1554560
trunk/ql/src/test/results/clientpositive/join34.q.out 1554560
trunk/ql/src/test/results/clientpositive/join35.q.out 1554560
trunk/ql/src/test/results/clientpositive/join9.q.out 1554560
trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out 1554560
trunk/ql/src/test/results/clientpositive/list_bucket_dml_10.q.out 1554560
trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out 1554560
trunk/ql/src/test/results/clientpositive/merge3.q.out 1554560
trunk/ql/src/test/results/clientpositive/pcr.q.out 1554560
trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out 1554560
trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample1.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample2.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample4.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample5.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample6.q.out 1554560
trunk/ql/src/test/results/clientpositive/sample7.q.out 1554560
trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out 1554560
trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out 1554560
trunk/ql/src/test/results/clientpositive/stats0.q.out 1554560
trunk/ql/src/test/results/clientpositive/stats11.q.out 1554560
trunk/ql/src/test/results/clientpositive/stats3.q.out 1554560
trunk/ql/src/test/results/clientpositive/union22.q.out 1554560
trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/groupby1.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input1.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input2.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input3.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input4.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input5.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input6.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input7.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input9.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/join1.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/join2.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/join3.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample2.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample3.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample4.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample5.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample6.q.xml 1554560
trunk/ql/src/test/results/compiler/plan/sample7.q.xml 1554560
Diff: https://reviews.apache.org/r/16526/diff/
Testing
-------
No new functionality. Existing regression suite suffices.
Thanks,
Ashutosh Chauhan
Re: Review Request 16526: more usage of paths
Posted by Ashutosh Chauhan <ha...@apache.org>.
> On Dec. 31, 2013, 6:40 a.m., Xuefu Zhang wrote:
> > trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java, line 503
> > <https://reviews.apache.org/r/16526/diff/1/?file=406579#file406579line503>
> >
> > There might be an tab/indention problem.
will take care of this one.
> On Dec. 31, 2013, 6:40 a.m., Xuefu Zhang wrote:
> > trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java, line 74
> > <https://reviews.apache.org/r/16526/diff/1/?file=406581#file406581line74>
> >
> > Is it safer to do path.toUri().toString()? I saw it's done this way in other part of changes.
Actually, I had path.toUri().toString() in my first version of patch, that resulted in few failures. On investigation, I found out pathToPartitionInfo() map which is <String,PartitionDesc> assumes path.toString() in key (not path.toUri().toString()). This is evident from HiveFileFormatUtils#getPartitionDescFromPathRecursively(). Best way to avoid this confusion is to have partToPartitionInfo() in <Path,PartitionDesc> form but thats a bigger change since that datastructure is used all over the place. I plan to make that change in next patch in this series.
> On Dec. 31, 2013, 6:40 a.m., Xuefu Zhang wrote:
> > trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java, line 180
> > <https://reviews.apache.org/r/16526/diff/1/?file=406588#file406588line180>
> >
> > Same as above.
This is actually other way around. inputPaths() was List<String>, now its List<Path>
> On Dec. 31, 2013, 6:40 a.m., Xuefu Zhang wrote:
> > trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java, line 190
> > <https://reviews.apache.org/r/16526/diff/1/?file=406588#file406588line190>
> >
> > Same as above.
This is actually other way around. inputPaths() was List<String>, now its List<Path>
> On Dec. 31, 2013, 6:40 a.m., Xuefu Zhang wrote:
> > trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java, line 35
> > <https://reviews.apache.org/r/16526/diff/1/?file=406608#file406608line35>
> >
> > Could you confirm that tmpDir isn't used? The old code seemed spending quite some effort generating the temp directory.
Yup, I did verify thats its not used anywhere. Pure dead code. Even worse, it was made part of explain unnecessarily, which resulted in changes of all those .q.out files.
- Ashutosh
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/16526/#review30995
-----------------------------------------------------------
On Dec. 31, 2013, 12:42 a.m., Ashutosh Chauhan wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/16526/
> -----------------------------------------------------------
>
> (Updated Dec. 31, 2013, 12:42 a.m.)
>
>
> Review request for hive, Xuefu Zhang and Xuefu Zhang.
>
>
> Bugs: HIVE-6121
> https://issues.apache.org/jira/browse/HIVE-6121
>
>
> Repository: hive
>
>
> Description
> -------
>
> Refactoring patch.
>
>
> Diffs
> -----
>
> trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
> trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadMultiFilesDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java 1554326
> trunk/ql/src/test/results/clientpositive/binary_output_format.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/input_part1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/input_part2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join17.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join26.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join32.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join33.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join34.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join35.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join9.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/list_bucket_dml_10.q.out 1554326
> trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out 1554326
> trunk/ql/src/test/results/clientpositive/merge3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/pcr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample6.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample7.q.out 1554326
> trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out 1554326
> trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats0.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats11.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/union22.q.out 1554326
> trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/groupby1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input4.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input5.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input6.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input7.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input9.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample4.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample5.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample6.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample7.q.xml 1554326
>
> Diff: https://reviews.apache.org/r/16526/diff/
>
>
> Testing
> -------
>
> No new functionality. Existing regression suite suffices.
>
>
> Thanks,
>
> Ashutosh Chauhan
>
>
Re: Review Request 16526: more usage of paths
Posted by Xuefu Zhang <xz...@cloudera.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/16526/#review30995
-----------------------------------------------------------
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
<https://reviews.apache.org/r/16526/#comment59314>
There might be an tab/indention problem.
trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java
<https://reviews.apache.org/r/16526/#comment59316>
Is it safer to do path.toUri().toString()? I saw it's done this way in other part of changes.
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
<https://reviews.apache.org/r/16526/#comment59319>
Same as above.
trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
<https://reviews.apache.org/r/16526/#comment59320>
Same as above.
trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
<https://reviews.apache.org/r/16526/#comment59341>
Could you confirm that tmpDir isn't used? The old code seemed spending quite some effort generating the temp directory.
- Xuefu Zhang
On Dec. 31, 2013, 12:42 a.m., Ashutosh Chauhan wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/16526/
> -----------------------------------------------------------
>
> (Updated Dec. 31, 2013, 12:42 a.m.)
>
>
> Review request for hive, Xuefu Zhang and Xuefu Zhang.
>
>
> Bugs: HIVE-6121
> https://issues.apache.org/jira/browse/HIVE-6121
>
>
> Repository: hive
>
>
> Description
> -------
>
> Refactoring patch.
>
>
> Diffs
> -----
>
> trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
> trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ArchiveWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadMultiFilesDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java 1554326
> trunk/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java 1554326
> trunk/ql/src/test/results/clientpositive/binary_output_format.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucket5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out 1554326
> trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out 1554326
> trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/input_part1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/input_part2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join17.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join26.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join32.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join33.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join34.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join35.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join9.q.out 1554326
> trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/list_bucket_dml_10.q.out 1554326
> trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out 1554326
> trunk/ql/src/test/results/clientpositive/merge3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/pcr.q.out 1554326
> trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample1.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample2.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample4.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample5.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample6.q.out 1554326
> trunk/ql/src/test/results/clientpositive/sample7.q.out 1554326
> trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out 1554326
> trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats0.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats11.q.out 1554326
> trunk/ql/src/test/results/clientpositive/stats3.q.out 1554326
> trunk/ql/src/test/results/clientpositive/union22.q.out 1554326
> trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/groupby1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input4.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input5.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input6.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input7.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input9.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join1.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/join3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample2.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample3.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample4.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample5.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample6.q.xml 1554326
> trunk/ql/src/test/results/compiler/plan/sample7.q.xml 1554326
>
> Diff: https://reviews.apache.org/r/16526/diff/
>
>
> Testing
> -------
>
> No new functionality. Existing regression suite suffices.
>
>
> Thanks,
>
> Ashutosh Chauhan
>
>