You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/03/12 21:19:58 UTC
svn commit: r752996 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/plan/
ql/src/test/results/clientpositive/ ql/src/test/results/compiler/plan/
Author: zshao
Date: Thu Mar 12 20:19:57 2009
New Revision: 752996
URL: http://svn.apache.org/viewvc?rev=752996&view=rev
Log:
HIVE-339. Fix count distinct in 1 map-reduce job with map side aggregation. (Namit Jain via zshao)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Mar 12 20:19:57 2009
@@ -19,6 +19,9 @@
BUG FIXES
+ HIVE-339. Fix count distinct in 1 map-reduce job with map side
+ aggregation. (Namit Jain via zshao)
+
HIVE-271. Move test-udfs creation to build-common in order to
fix broken 0.17 build. (athusoo)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Mar 12 20:19:57 2009
@@ -407,10 +407,10 @@
// Total number of input rows is needed for hash aggregation only
if (hashAggr) {
numRowsInput++;
- // if hash aggregation is not behvaing properly, disable it
+ // if hash aggregation is not behaving properly, disable it
if (numRowsInput == numRowsCompareHashAggr) {
numRowsCompareHashAggr += groupbyMapAggrInterval;
- // map-side aggregation should reduce the entries by atleast half
+ // map-side aggregation should reduce the entries by at-least half
if ((numRowsHashTbl * 2) > numRowsInput) {
LOG.warn("Disable Hash Aggr: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput);
flush(true);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Thu Mar 12 20:19:57 2009
@@ -99,7 +99,7 @@
return parentOperators;
}
- protected String id;
+ transient protected String id;
protected T conf;
protected boolean done;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Mar 12 20:19:57 2009
@@ -1805,7 +1805,21 @@
* @param input
* @return
* @throws SemanticException
- */
+ *
+ * Generate a Group-By plan using 1 map-reduce job.
+ * Spray by the group by key, and sort by the distinct key (if any), and
+ * compute aggregates *
+ * The agggregation evaluation functions are as follows:
+ * Partitioning Key:
+ * grouping key
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: iterate/merge
+ * (mode = COMPLETE)
+ **/
@SuppressWarnings({ "unused", "nls" })
private Operator genGroupByPlan1MR(String dest, QB qb,
Operator input) throws SemanticException {
@@ -1839,6 +1853,36 @@
* @param input
* @return
* @throws SemanticException
+ *
+ * Generate a Group-By plan using a 2 map-reduce jobs.
+ * Spray by the grouping key and distinct key (or a random number, if no distinct is
+ * present) in hope of getting a uniform distribution, and compute partial aggregates
+ * grouped by the reduction key (grouping key + distinct key).
+ * Evaluate partial aggregates first, and spray by the grouping key to compute actual
+ * aggregates in the second phase.
+ * The agggregation evaluation functions are as follows:
+ * Partitioning Key:
+ * random() if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: iterate/terminatePartial
+ * (mode = PARTIAL1)
+ *
+ * STAGE 2
+ *
+ * Partitioning Key:
+ * grouping key
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: merge/terminate
+ * (mode = FINAL)
*/
@SuppressWarnings("nls")
private Operator genGroupByPlan2MR(String dest, QB qb,
@@ -1883,11 +1927,26 @@
}
/**
- * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map
- * side partial aggregation (to reduce the amount of data). Then spray by
- * the distinct key (or a random number) in hope of getting a uniform
- * distribution, and compute partial aggregates grouped by that distinct key.
- * Evaluate partial aggregates first, followed by actual aggregates.
+ * Generate a Group-By plan using 1 map-reduce job.
+ * First perform a map-side partial aggregation (to reduce the amount of data), at this
+ * point of time, we may turn off map-side partial aggregation based on its performance.
+ * Then spray by the group by key, and sort by the distinct key (if any), and
+ * compute aggregates based on actual aggregates
+ *
+ * The agggregation evaluation functions are as follows:
+ * Mapper: iterate/terminatePartial
+ * (mode = HASH)
+ *
+ * Partitioning Key:
+ * grouping key
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: iterate/terminate if DISTINCT
+ * merge/terminate if NO DISTINCT
+ * (mode = MERGEPARTIAL)
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanMapAggr1MR(String dest, QB qb,
@@ -1910,15 +1969,51 @@
Operator reduceSinkOperatorInfo =
genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, getGroupByForClause(parseInfo, dest).size(), numReducers, true);
- return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL);
+ // This is a 1-stage map-reduce processing of the groupby. Tha map-side aggregates was just used to
+ // reduce output data. In case of distincts, partial results are not used, and so iterate is again
+ // invoked on the reducer. In case of non-distincts, partial results are used, and merge is invoked
+ // on the reducer.
+ return genGroupByPlanGroupByOperator1(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.MERGEPARTIAL);
}
/**
- * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map
- * side partial aggregation (to reduce the amount of data). Then spray by
- * the distinct key (or a random number) in hope of getting a uniform
- * distribution, and compute partial aggregates grouped by that distinct key.
- * Evaluate partial aggregates first, followed by actual aggregates.
+ * Generate a Group-By plan using a 2 map-reduce jobs.
+ * However, only 1 group-by plan is generated if the query involves no grouping key and
+ * no distincts. In that case, the plan is same as generated by genGroupByPlanMapAggr1MR.
+ * Otherwise, the following plan is generated:
+ * First perform a map side partial aggregation (to reduce the amount of data). Then
+ * spray by the grouping key and distinct key (or a random number, if no distinct is
+ * present) in hope of getting a uniform distribution, and compute partial aggregates
+ * grouped by the reduction key (grouping key + distinct key).
+ * Evaluate partial aggregates first, and spray by the grouping key to compute actual
+ * aggregates in the second phase.
+ * The agggregation evaluation functions are as follows:
+ * Mapper: iterate/terminatePartial
+ * (mode = HASH)
+ *
+ * Partitioning Key:
+ * random() if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: iterate/terminatePartial if DISTINCT
+ * merge/terminatePartial if NO DISTINCT
+ * (mode = PARTIAL2)
+ *
+ * STAGE 2
+ *
+ * Partitioining Key:
+ * grouping key
+ *
+ * Sorting Key:
+ * grouping key if no DISTINCT
+ * grouping + distinct key if DISTINCT
+ *
+ * Reducer: merge/terminate
+ * (mode = FINAL)
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanMapAggr2MR(String dest, QB qb,
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java Thu Mar 12 20:19:57 2009
@@ -28,7 +28,7 @@
* HASH: the same as PARTIAL1 but use hash-table-based aggregation
*/
private static final long serialVersionUID = 1L;
- public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, FINAL, HASH };
+ public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, FINAL, HASH, MERGEPARTIAL };
private Mode mode;
private java.util.ArrayList<exprNodeDesc> keys;
private java.util.ArrayList<org.apache.hadoop.hive.ql.plan.aggregationDesc> aggregators;
@@ -63,6 +63,8 @@
return "hash";
case FINAL:
return "final";
+ case MERGEPARTIAL:
+ return "mergepartial";
}
return "unknown";
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
Files hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out (original) and hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out Thu Mar 12 20:19:57 2009 differ
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out Thu Mar 12 20:19:57 2009
@@ -37,7 +37,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
@@ -56,7 +56,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /data/users/njain/hive1/hive/build/ql/tmp/262628633/486992722.10001
+ /data/users/njain/hive1/hive/build/ql/tmp/746802126/1233171241.10001
Reduce Output Operator
sort order:
tag: -1
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out Thu Mar 12 20:19:57 2009
@@ -36,7 +36,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out Thu Mar 12 20:19:57 2009
@@ -36,7 +36,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out Thu Mar 12 20:19:57 2009
@@ -36,7 +36,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out Thu Mar 12 20:19:57 2009
@@ -40,11 +40,11 @@
Group By Operator
aggregations:
expr: sum(VALUE.0)
- expr: count(VALUE.1)
+ expr: count(DISTINCT KEY.1)
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out Thu Mar 12 20:19:57 2009
@@ -47,10 +47,10 @@
aggregations:
expr: sum(VALUE.0)
expr: max(VALUE.1)
- expr: avg(VALUE.2)
+ expr: avg(DISTINCT UDFToDouble(KEY.0))
expr: min(VALUE.3)
expr: avg(VALUE.4)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out Thu Mar 12 20:19:57 2009
@@ -25,7 +25,7 @@
Group By Operator
aggregations:
expr: count(VALUE.0)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out Thu Mar 12 20:19:57 2009
@@ -28,7 +28,7 @@
Group By Operator
aggregations:
expr: sum(VALUE.0)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out Thu Mar 12 20:19:57 2009
@@ -33,7 +33,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out Thu Mar 12 20:19:57 2009
@@ -52,7 +52,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
@@ -87,7 +87,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /data/users/njain/hive1/hive/build/ql/tmp/123213583/1044345267.10002
+ /data/users/njain/hive1/hive/build/ql/tmp/272279570/1527190226.10002
Reduce Output Operator
key expressions:
expr: 0
@@ -107,7 +107,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out Thu Mar 12 20:19:57 2009
@@ -54,11 +54,11 @@
Reduce Operator Tree:
Group By Operator
aggregations:
- expr: count(VALUE.0)
+ expr: count(DISTINCT KEY.1)
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
@@ -93,7 +93,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /data/users/njain/hive1/hive/build/ql/tmp/1475395360/1425775.10002
+ /data/users/njain/hive1/hive/build/ql/tmp/1165802094/307152394.10002
Reduce Output Operator
key expressions:
expr: 0
@@ -111,11 +111,11 @@
Reduce Operator Tree:
Group By Operator
aggregations:
- expr: count(VALUE.0)
+ expr: count(DISTINCT KEY.1)
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out Thu Mar 12 20:19:57 2009
@@ -39,11 +39,11 @@
Reduce Operator Tree:
Group By Operator
aggregations:
- expr: count(VALUE.0)
+ expr: count(DISTINCT KEY.1)
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
@@ -167,7 +167,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out Thu Mar 12 20:19:57 2009
@@ -44,7 +44,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: '1234'
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out Thu Mar 12 20:19:57 2009
@@ -44,7 +44,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: '1234'
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out Thu Mar 12 20:19:57 2009
@@ -40,7 +40,7 @@
keys:
expr: KEY.0
type: string
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out Thu Mar 12 20:19:57 2009
@@ -41,7 +41,7 @@
expr: count(VALUE.2)
expr: sum(VALUE.3)
expr: min(VALUE.4)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 2
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out Thu Mar 12 20:19:57 2009
@@ -32,7 +32,7 @@
expr: sum(VALUE.0)
expr: count(VALUE.1)
expr: avg(VALUE.2)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 2
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out Thu Mar 12 20:19:57 2009
@@ -47,7 +47,7 @@
Group By Operator
aggregations:
expr: count(VALUE.0)
- mode: final
+ mode: mergepartial
Select Operator
expressions:
expr: 0
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml Thu Mar 12 20:19:57 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1045332593/216206529.10000.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/434688883/186307528.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -584,7 +584,7 @@
<int>1</int>
</void>
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1045332593/216206529.10000.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/434688883/186307528.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -739,7 +739,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml Thu Mar 12 20:19:57 2009
@@ -711,7 +711,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/717653767/1509670805.10001.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/10857281/717275265.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -920,7 +920,7 @@
<string>merge</string>
</void>
<void method="add">
- <string>merge</string>
+ <string>iterate</string>
</void>
</object>
</void>
@@ -952,15 +952,18 @@
<void property="aggregationClass">
<class>org.apache.hadoop.hive.ql.udf.UDAFCount</class>
</void>
+ <void property="distinct">
+ <boolean>true</boolean>
+ </void>
<void property="parameters">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc">
<void property="column">
- <string>VALUE.1</string>
+ <string>KEY.1</string>
</void>
<void property="typeInfo">
- <object idref="PrimitiveTypeInfo2"/>
+ <object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
@@ -996,7 +999,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml Thu Mar 12 20:19:57 2009
@@ -997,7 +997,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/334497369.10001.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/150776966.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -1188,7 +1188,7 @@
<string>merge</string>
</void>
<void method="add">
- <string>merge</string>
+ <string>iterate</string>
</void>
<void method="add">
<string>merge</string>
@@ -1247,15 +1247,42 @@
<void property="aggregationClass">
<class>org.apache.hadoop.hive.ql.udf.UDAFAvg</class>
</void>
+ <void property="distinct">
+ <boolean>true</boolean>
+ </void>
<void property="parameters">
<object class="java.util.ArrayList">
<void method="add">
- <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc">
- <void property="column">
- <string>VALUE.2</string>
+ <object class="org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc">
+ <void property="UDFClass">
+ <class>org.apache.hadoop.hive.ql.udf.UDFToDouble</class>
+ </void>
+ <void property="UDFMethod">
+ <object class="org.apache.hadoop.hive.ql.udf.UDFToDouble" method="getMethod">
+ <string>evaluate</string>
+ <array class="java.lang.Class" length="1">
+ <void index="0">
+ <class>java.lang.String</class>
+ </void>
+ </array>
+ </object>
+ </void>
+ <void property="children">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc">
+ <void property="column">
+ <string>KEY.0</string>
+ </void>
+ <void property="typeInfo">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ </object>
</void>
<void property="typeInfo">
- <object idref="PrimitiveTypeInfo0"/>
+ <object idref="PrimitiveTypeInfo1"/>
</void>
</object>
</void>
@@ -1331,7 +1358,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml Thu Mar 12 20:19:57 2009
@@ -457,7 +457,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/69474526.10001.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/57949151.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -584,7 +584,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml Thu Mar 12 20:19:57 2009
@@ -479,7 +479,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1489470041/391886937.10001.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/979288592/53411655.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -656,7 +656,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml Thu Mar 12 20:19:57 2009
@@ -457,7 +457,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/889541974.10001.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/272705795.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -584,7 +584,7 @@
</void>
<void property="mode">
<object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">
- <string>FINAL</string>
+ <string>MERGEPARTIAL</string>
</object>
</void>
</object>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml Thu Mar 12 20:19:57 2009
@@ -28,7 +28,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/752613561/196516335.10000.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/672033374/77772175.10000.insclause-0</string>
</void>
<void property="targetDir">
<string>../build/ql/test/data/warehouse/union.out</string>
@@ -82,7 +82,7 @@
<int>1</int>
</void>
<void property="dirName">
- <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/752613561/196516335.10000.insclause-0</string>
+ <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/672033374/77772175.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">