You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/28 22:34:39 UTC
svn commit: r1476829 - in /hive/branches/branch-0.11/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/
java/org/apache/hadoop/hive/ql/plan/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Sun Apr 28 20:34:38 2013
New Revision: 1476829
URL: http://svn.apache.org/r1476829
Log:
HIVE-4358 : Check for Map side processing in PTFOp is no longer valid (Harish Butani via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q
hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Sun Apr 28 20:34:38 2013
@@ -72,14 +72,7 @@ public class PTFOperator extends Operato
hiveConf = new HiveConf(jobConf, PTFOperator.class);
// if the parent is ExtractOperator, this invocation is from reduce-side
Operator<? extends OperatorDesc> parentOp = getParentOperators().get(0);
- if (parentOp instanceof ExtractOperator)
- {
- isMapOperator = false;
- }
- else
- {
- isMapOperator = true;
- }
+ isMapOperator = conf.isMapSide();
reconstructQueryDef(hiveConf);
inputPart = createFirstPartitionForChain(
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sun Apr 28 20:34:38 2013
@@ -10631,6 +10631,7 @@ public class SemanticAnalyzer extends Ba
{
RowResolver ptfMapRR = tabDef.getRawInputShape().getRr();
+ ptfDesc.setMapSide(true);
input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
new RowSchema(ptfMapRR.getColumnInfos()),
input), ptfMapRR);
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java Sun Apr 28 20:34:38 2013
@@ -46,6 +46,10 @@ public class PTFDesc extends AbstractOpe
PartitionedTableFunctionDef funcDef;
LeadLagInfo llInfo;
+ /*
+ * is this PTFDesc for a Map-Side PTF Operation?
+ */
+ boolean isMapSide = false;
static{
PTFUtils.makeTransient(PTFDesc.class, "llInfo");
@@ -75,6 +79,14 @@ public class PTFDesc extends AbstractOpe
return funcDef != null && (funcDef instanceof WindowTableFunctionDef);
}
+ public boolean isMapSide() {
+ return isMapSide;
+ }
+
+ public void setMapSide(boolean isMapSide) {
+ this.isMapSide = isMapSide;
+ }
+
public abstract static class PTFInputDef {
String expressionTreeString;
ShapeDetails outputShape;
Modified: hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q (original)
+++ hive/branches/branch-0.11/ql/src/test/queries/clientpositive/windowing.q Sun Apr 28 20:34:38 2013
@@ -224,6 +224,11 @@ window w1 as (distribute by p_mfgr sort
) sq
order by p_mfgr, p_brand;
+select p_mfgr, p_brand, s,
+round(sum(s),2) over w1 as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row);
+
-- 23. testCreateViewWithWindowingQuery
create view IF NOT EXISTS mfgr_brand_price_view as
select p_mfgr, p_brand,
Modified: hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out?rev=1476829&r1=1476828&r2=1476829&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out (original)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/windowing.q.out Sun Apr 28 20:34:38 2013
@@ -1066,6 +1066,38 @@ Manufacturer#4 Brand#42 2581.68 7337.62
Manufacturer#5 Brand#51 1611.66 7672.66
Manufacturer#5 Brand#52 3254.17 7672.66
Manufacturer#5 Brand#53 2806.83 7672.66
+PREHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s),2) over w1 as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s),2) over w1 as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4800.84 4800.84
+Manufacturer#1 Brand#14 2346.3 7147.14
+Manufacturer#1 Brand#15 1602.59 8749.73
+Manufacturer#2 Brand#22 3491.38 3491.38
+Manufacturer#2 Brand#23 2031.98 5523.36
+Manufacturer#2 Brand#24 1698.66 7222.02
+Manufacturer#2 Brand#25 1701.6 5432.24
+Manufacturer#3 Brand#31 1671.68 1671.68
+Manufacturer#3 Brand#32 3333.37 5005.05
+Manufacturer#3 Brand#34 1337.29 6342.34
+Manufacturer#3 Brand#35 1190.27 5860.93
+Manufacturer#4 Brand#41 4755.94 4755.94
+Manufacturer#4 Brand#42 2581.68 7337.62
+Manufacturer#5 Brand#51 1611.66 1611.66
+Manufacturer#5 Brand#52 3254.17 4865.83
+Manufacturer#5 Brand#53 2806.83 7672.66
PREHOOK: query: -- 23. testCreateViewWithWindowingQuery
create view IF NOT EXISTS mfgr_brand_price_view as
select p_mfgr, p_brand,