You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/02 16:16:37 UTC
svn commit: r1463556 [1/15] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ data/files/ ql/if/
ql/src/gen/thrift/gen-cpp/
ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/
ql/src/gen/thrift/gen-php/ ql/src/gen/thrift/gen...
Author: hashutosh
Date: Tue Apr 2 14:16:34 2013
New Revision: 1463556
URL: http://svn.apache.org/r1463556
Log:
HIVE-896 : Add LEAD/LAG/FIRST/LAST analytical windowing functions to Hive. (Harish Butani via Ashutosh Chauhan)
Added:
hive/trunk/data/files/flights_tiny.txt
hive/trunk/data/files/over10k
hive/trunk/data/files/part.rc (with props)
hive/trunk/data/files/part.seq (with props)
hive/trunk/data/files/part_tiny.txt
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/PartitionTableFunctionDescription.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionDescription.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCumeDist.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFDenseRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLag.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLead.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFNTile.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRank.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFRowNumber.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_DistributeByOrderBy.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_DuplicateWindowAlias.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_JoinWithAmbigousAlias.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_PartitionBySortBy.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_window_boundaries.q
hive/trunk/ql/src/test/queries/clientnegative/ptf_window_boundaries2.q
hive/trunk/ql/src/test/queries/clientpositive/leadlag.q
hive/trunk/ql/src/test/queries/clientpositive/leadlag_queries.q
hive/trunk/ql/src/test/queries/clientpositive/ptf.q
hive/trunk/ql/src/test/queries/clientpositive/ptf_general_queries.q
hive/trunk/ql/src/test/queries/clientpositive/ptf_npath.q
hive/trunk/ql/src/test/queries/clientpositive/ptf_rcfile.q
hive/trunk/ql/src/test/queries/clientpositive/ptf_seqfile.q
hive/trunk/ql/src/test/queries/clientpositive/windowing.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_columnPruning.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_expressions.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_multipartitioning.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_navfn.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_ntile.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_rank.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_windowspec.q
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_AggrFuncsWithNoGBYNoPartDef.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_AmbiguousWindowDefn.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_DistributeByOrderBy.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_DuplicateWindowAlias.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_HavingLeadWithPTF.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_JoinWithAmbigousAlias.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_PartitionBySortBy.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_negative_WhereWithRankCond.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_window_boundaries.q.out
hive/trunk/ql/src/test/results/clientnegative/ptf_window_boundaries2.q.out
hive/trunk/ql/src/test/results/clientpositive/leadlag.q.out
hive/trunk/ql/src/test/results/clientpositive/leadlag_queries.q.out
hive/trunk/ql/src/test/results/clientpositive/ptf.q.out
hive/trunk/ql/src/test/results/clientpositive/ptf_general_queries.q.out
hive/trunk/ql/src/test/results/clientpositive/ptf_npath.q.out
hive/trunk/ql/src/test/results/clientpositive/ptf_rcfile.q.out
hive/trunk/ql/src/test/results/clientpositive/ptf_seqfile.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_columnPruning.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_expressions.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_navfn.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_ntile.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_rank.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_udaf.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_windowspec.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/if/queryplan.thrift
hive/trunk/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
hive/trunk/ql/src/gen/thrift/gen-cpp/queryplan_types.h
hive/trunk/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Operator.java
hive/trunk/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
hive/trunk/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Query.java
hive/trunk/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Stage.java
hive/trunk/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/Task.java
hive/trunk/ql/src/gen/thrift/gen-php/Types.php
hive/trunk/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
hive/trunk/ql/src/gen/thrift/gen-rb/queryplan_types.rb
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1463556&r1=1463555&r2=1463556&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Apr 2 14:16:34 2013
@@ -760,6 +760,12 @@ public class HiveConf extends Configurat
// Whether to show the unquoted partition names in query results.
HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false),
+
+ // ptf partition constants
+ HIVE_PTF_PARTITION_PERSISTENCE_CLASS("hive.ptf.partition.persistence",
+ "org.apache.hadoop.hive.ql.exec.PTFPersistence$PartitionedByteBasedList"),
+ HIVE_PTF_PARTITION_PERSISTENT_SIZE("hive.ptf.partition.persistence.memsize",
+ (int) Math.pow(2, (5 + 10 + 10)) ), // 32MB
;
public final String varname;
Added: hive/trunk/data/files/flights_tiny.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/flights_tiny.txt?rev=1463556&view=auto
==============================================================================
--- hive/trunk/data/files/flights_tiny.txt (added)
+++ hive/trunk/data/files/flights_tiny.txt Tue Apr 2 14:16:34 2013
@@ -0,0 +1,137 @@
+BaltimoreNew York20101020-30.01064
+BaltimoreNew York2010102023.01142
+BaltimoreNew York201010206.01599
+ChicagoNew York2010102042.0361
+ChicagoNew York2010102024.0897
+ChicagoNew York2010102015.01531
+ChicagoNew York20101020-6.01610
+ChicagoNew York20101020-2.03198
+BaltimoreNew York2010102117.01064
+BaltimoreNew York20101021105.01142
+BaltimoreNew York2010102128.01599
+ChicagoNew York20101021142.0361
+ChicagoNew York2010102177.0897
+ChicagoNew York2010102153.01531
+ChicagoNew York20101021-5.01610
+ChicagoNew York2010102151.03198
+BaltimoreNew York20101022-12.01064
+BaltimoreNew York2010102254.01142
+BaltimoreNew York2010102218.01599
+ChicagoNew York201010222.0361
+ChicagoNew York2010102224.0897
+ChicagoNew York2010102216.01531
+ChicagoNew York20101022-6.01610
+ChicagoNew York20101022-11.03198
+BaltimoreNew York2010102318.0272
+BaltimoreNew York20101023-10.01805
+BaltimoreNew York201010236.03171
+ChicagoNew York201010233.0384
+ChicagoNew York2010102332.0426
+ChicagoNew York201010231.0650
+ChicagoNew York2010102311.03085
+BaltimoreNew York2010102412.01599
+BaltimoreNew York2010102420.02571
+ChicagoNew York2010102410.0361
+ChicagoNew York20101024113.0897
+ChicagoNew York20101024-5.01531
+ChicagoNew York20101024-17.01610
+ChicagoNew York20101024-3.03198
+BaltimoreNew York20101025-25.01064
+BaltimoreNew York2010102592.01142
+BaltimoreNew York20101025106.01599
+ChicagoNew York2010102531.0361
+ChicagoNew York20101025-1.0897
+ChicagoNew York2010102543.01531
+ChicagoNew York201010256.01610
+ChicagoNew York20101025-16.03198
+BaltimoreNew York20101026-22.01064
+BaltimoreNew York20101026123.01142
+BaltimoreNew York2010102690.01599
+ChicagoNew York2010102612.0361
+ChicagoNew York201010260.0897
+ChicagoNew York2010102629.01531
+ChicagoNew York20101026-17.01610
+ChicagoNew York201010266.03198
+BaltimoreNew York20101027-18.01064
+BaltimoreNew York2010102749.01142
+BaltimoreNew York2010102792.01599
+ChicagoNew York20101027148.0361
+ChicagoNew York20101027-11.0897
+ChicagoNew York2010102770.01531
+ChicagoNew York201010278.01610
+ChicagoNew York2010102721.03198
+BaltimoreNew York20101028-4.01064
+BaltimoreNew York20101028-14.01142
+BaltimoreNew York20101028-14.01599
+ChicagoNew York201010282.0361
+ChicagoNew York201010282.0897
+ChicagoNew York20101028-11.01531
+ChicagoNew York201010283.01610
+ChicagoNew York20101028-18.03198
+BaltimoreNew York20101029-24.01064
+BaltimoreNew York2010102921.01142
+BaltimoreNew York20101029-2.01599
+ChicagoNew York20101029-12.0361
+ChicagoNew York20101029-11.0897
+ChicagoNew York2010102915.01531
+ChicagoNew York20101029-18.01610
+ChicagoNew York20101029-4.03198
+BaltimoreNew York2010103014.0272
+BaltimoreNew York20101030-1.01805
+BaltimoreNew York201010305.03171
+ChicagoNew York20101030-6.0384
+ChicagoNew York20101030-10.0426
+ChicagoNew York20101030-5.0650
+ChicagoNew York20101030-5.03085
+BaltimoreNew York20101031-1.01599
+BaltimoreNew York20101031-14.02571
+ChicagoNew York20101031-25.0361
+ChicagoNew York20101031-18.0897
+ChicagoNew York20101031-4.01531
+ChicagoNew York20101031-22.01610
+ChicagoNew York20101031-15.03198
+ClevelandNew York20101030-23.02018
+ClevelandNew York20101030-12.02932
+ClevelandNew York20101029-4.02630
+ClevelandNew York20101029-19.02646
+ClevelandNew York20101029-12.03014
+ClevelandNew York201010283.02630
+ClevelandNew York20101028-6.02646
+ClevelandNew York201010281.03014
+ClevelandNew York2010102716.02630
+ClevelandNew York2010102727.03014
+ClevelandNew York201010264.02630
+ClevelandNew York20101026-27.02646
+ClevelandNew York20101026-11.02662
+ClevelandNew York2010102613.03014
+ClevelandNew York20101025-4.02630
+ClevelandNew York2010102581.02646
+ClevelandNew York2010102542.03014
+ClevelandNew York201010245.02254
+ClevelandNew York20101024-11.02630
+ClevelandNew York20101024-20.02646
+ClevelandNew York20101024-9.03014
+ClevelandNew York20101023-21.02932
+ClevelandNew York201010221.02630
+ClevelandNew York20101022-25.02646
+ClevelandNew York20101022-3.03014
+ClevelandNew York201010213.02630
+ClevelandNew York2010102129.02646
+ClevelandNew York2010102172.03014
+ClevelandNew York20101020-8.02630
+ClevelandNew York20101020-15.03014
+WashingtonNew York20101023-25.05832
+WashingtonNew York20101023-21.05904
+WashingtonNew York20101023-18.05917
+WashingtonNew York20101030-27.05904
+WashingtonNew York20101030-16.05917
+WashingtonNew York20101020-2.07291
+WashingtonNew York2010102122.07291
+WashingtonNew York20101023-16.07274
+WashingtonNew York20101024-26.07282
+WashingtonNew York201010259.07291
+WashingtonNew York201010264.07291
+WashingtonNew York2010102726.07291
+WashingtonNew York2010102845.07291
+WashingtonNew York201010291.07291
+WashingtonNew York20101031-18.07282