You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/03/20 18:03:54 UTC
svn commit: r1458953 - in /hive/branches/ptf-windowing/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientnegative/
test/queries/clientpositive/ test/results/clientnegative/
test/results/clientpos...
Author: hashutosh
Date: Wed Mar 20 17:03:54 2013
New Revision: 1458953
URL: http://svn.apache.org/r1458953
Log:
HIVE-4041 [jira] Support multiple partitionings in a single Query
(Harish Butani via Ashutosh Chauhan)
Summary:
add support for multiple partitioning
Currently we disallow queries if the partition specifications of all Wdw fns are not the same. We can relax this by generating multiple PTFOps based on the unique partitionings in a Query. For partitionings that only differ in sort, we can introduce a sort step in between PTFOps, which can happen in the same Reduce task.
Test Plan: added tests to windowing_multipartitioning.q
Reviewers: JIRA, ashutoshc
Reviewed By: ashutoshc
Differential Revision: https://reviews.facebook.net/D9381
Added:
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java
Removed:
hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleDistributeClause.q
hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q
hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q
hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleSortClause.q
hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleDistributeClause.q.out
hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q.out
hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q.out
hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleSortClause.q.out
Modified:
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q
hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out
hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Wed Mar 20 17:03:54 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -208,12 +209,12 @@ public final class ColumnPrunerProcFacto
private static RowResolver buildPrunedRR(List<String> prunedCols,
RowResolver oldRR, ArrayList<ColumnInfo> sig) throws SemanticException{
RowResolver newRR = new RowResolver();
- for (String col : prunedCols) {
- String[] nm = oldRR.reverseLookup(col);
- ColumnInfo colInfo = oldRR.get(nm[0], nm[1]);
- if (colInfo != null) {
- newRR.put(nm[0], nm[1], colInfo);
- sig.add(colInfo);
+ HashSet<String> prunedColsSet = new HashSet<String>(prunedCols);
+ for(ColumnInfo cInfo : oldRR.getRowSchema().getSignature()) {
+ if ( prunedColsSet.contains(cInfo.getInternalName())) {
+ String[] nm = oldRR.reverseLookup(cInfo.getInternalName());
+ newRR.put(nm[0], nm[1], cInfo);
+ sig.add(cInfo);
}
}
return newRR;
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java Wed Mar 20 17:03:54 2013
@@ -339,7 +339,7 @@ public class PTFInvocationSpec {
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((expression == null) ? 0 : expression.hashCode());
+ result = prime * result + ((expression == null) ? 0 : expression.toStringTree().hashCode());
return result;
}
@@ -351,7 +351,7 @@ public class PTFInvocationSpec {
if (obj == null) {
return false;
}
- if (getClass() != obj.getClass()) {
+ if (!getClass().isAssignableFrom(obj.getClass())) {
return false;
}
PartitionExpression other = (PartitionExpression) obj;
@@ -408,6 +408,38 @@ public class PTFInvocationSpec {
expressions.add(c);
}
+ protected boolean isPrefixedBy(PartitionSpec pSpec) {
+ if ( pSpec == null || pSpec.getExpressions() == null) {
+ return true;
+ }
+
+ int pExprCnt = pSpec.getExpressions().size();
+ int exprCnt = getExpressions() == null ? 0 : getExpressions().size();
+
+ if ( exprCnt < pExprCnt ) {
+ return false;
+ }
+
+ for(int i=0; i < pExprCnt; i++) {
+ if ( !pSpec.getExpressions().get(i).equals(getExpressions().get(i)) ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ protected void prefixBy(PartitionSpec pSpec) {
+ if ( pSpec == null || pSpec.getExpressions() == null) {
+ return;
+ }
+ if ( expressions == null ) {
+ expressions = new ArrayList<PTFInvocationSpec.OrderExpression>();
+ }
+ for(int i = pSpec.getExpressions().size() - 1; i >= 0; i--) {
+ expressions.add(0, new OrderExpression(pSpec.getExpressions().get(i)));
+ }
+ }
+
@Override
public int hashCode()
{
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Wed Mar 20 17:03:54 2013
@@ -477,6 +477,14 @@ public class PTFTranslator {
return;
}
PartitionDef partDef = translate(def.getRawInputShape(), spec.getPartition());
+
+ if ( spec.getOrder() == null ) {
+ spec.setOrder(new OrderSpec());
+ }
+ if ( !spec.getOrder().isPrefixedBy(spec.getPartition()) ) {
+ spec.getOrder().prefixBy(spec.getPartition());
+ }
+
OrderDef orderDef = translate(def.getRawInputShape(), spec.getOrder(), partDef);
def.setPartition(partDef);
def.setOrder(orderDef);
@@ -535,19 +543,6 @@ public class PTFTranslator {
OrderSpec spec,
PartitionDef partitionDef) throws SemanticException {
- if (spec == null || spec.getExpressions() == null || spec.getExpressions().size() == 0)
- {
- if (partitionDef == null) {
- return null;
- }
- return new OrderDef(partitionDef);
- }
-
- if (partitionDef == null) {
- throw new SemanticException(String.format(
- "Cannot have an Order spec (%s) w/o a Partition spec", spec));
- }
-
OrderDef def = new OrderDef();
for (OrderExpression oExpr : spec.getExpressions())
{
@@ -555,46 +550,6 @@ public class PTFTranslator {
def.addExpression(oexpDef);
}
- /*
- * either all partition expressions must be in Order list or none must be specified.
- * If none are specified then add them all.
- */
- int numOfPartColumns = 0;
- List<OrderExpressionDef> orderExprs = def.getExpressions();
- List<PTFExpressionDef> partExprs = partitionDef.getExpressions();
- int chkSize = partExprs.size();
- chkSize = chkSize > orderExprs.size() ? orderExprs.size() : chkSize;
- for (int i = 0; i < chkSize; i++)
- {
- if (orderExprs.get(i).getExpressionTreeString()
- .equals(partExprs.get(i).getExpressionTreeString()))
- {
- numOfPartColumns++;
- } else {
- break;
- }
- }
-
- if (numOfPartColumns != 0 && numOfPartColumns != partExprs.size())
- {
- throw new SemanticException(
- String.format(
- "all partition columns must be in order clause(%s) or none should be specified",
- spec));
- }
-
- if (numOfPartColumns == 0)
- {
- ArrayList<OrderExpressionDef> combinedOrderCols = new ArrayList<OrderExpressionDef>();
- for (PTFExpressionDef eDef : partExprs)
- {
- OrderExpressionDef oeDef = new OrderExpressionDef(eDef);
- combinedOrderCols.add(oeDef);
- }
- combinedOrderCols.addAll(orderExprs);
- def.setExpressions(combinedOrderCols);
- }
-
return def;
}
@@ -620,12 +575,12 @@ public class PTFTranslator {
return oexpDef;
}
- private WindowFrameDef translate(String wFnName, ShapeDetails inpShape, WindowSpec spec) throws SemanticException {
- PartitionSpec pSpec = spec.getPartition();
- OrderSpec oSpec = spec.getOrder();
- PartitionDef pDef = translate(inpShape, pSpec);
- OrderDef oDef = translate(inpShape, oSpec, pDef);
- validatePartitioningForWdwFn(wFnName, pDef, oDef);
+ private WindowFrameDef translate(String wFnName, ShapeDetails inpShape, WindowSpec spec)
+ throws SemanticException {
+ /*
+ * Since we componentize Windowing, no need to translate
+ * the Partition & Order specs of individual WFns.
+ */
return translate(inpShape, spec.getWindowFrame());
}
@@ -740,25 +695,6 @@ public class PTFTranslator {
}
- private void validatePartitioningForWdwFn(String wFnName, PartitionDef fPart, OrderDef fOrder)
- throws SemanticException
- {
- PartitionDef tPart = ptfDesc.getFuncDef().getPartition();
-
- if (!PTFTranslator.isCompatible(tPart, fPart))
- {
- throw new SemanticException(
- String.format("Window Function '%s' has an incompatible partition clause", wFnName));
- }
-
- OrderDef tOrder = ptfDesc.getFuncDef().getOrder();
- if (!PTFTranslator.isCompatible(tOrder, fOrder))
- {
- throw new SemanticException(
- String.format("Window Function '%s' has an incompatible order clause", wFnName));
- }
- }
-
private ShapeDetails setupTableFnShape(String fnName, ShapeDetails inpShape, StructObjectInspector OI, ArrayList<String> columnNames, RowResolver rr)
throws SemanticException {
if (fnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION)
@@ -956,84 +892,6 @@ public class PTFTranslator {
}
/*
- * Partition & Order expression compatibility methods
- */
-
- protected static boolean isCompatible(PartitionDef p1, PartitionDef p2) {
- if (p1 == null && p2 == null) {
- return true;
- }
- if (p1 == null && p2 != null) {
- return false;
- }
- if (p1 != null && p2 == null) {
- return true;
- }
-
- ArrayList<PTFExpressionDef> p1Exprs = p1.getExpressions();
- ArrayList<PTFExpressionDef> p2Exprs = p2.getExpressions();
- if (p1Exprs.size() != p2Exprs.size()) {
- return false;
- }
- for (int i = 0; i < p1Exprs.size(); i++)
- {
- boolean e = isEqual(p1Exprs.get(i), p2Exprs.get(i));
- if (!e) {
- return false;
- }
- }
- return true;
- }
-
- protected static boolean isEqual(PTFExpressionDef e1, PTFExpressionDef e2) {
- if (e1 == null && e2 == null) {
- return false;
- }
- if (e1 == null && e2 != null) {
- return false;
- }
- if (e1 != null && e2 == null) {
- return false;
- }
-
- return e1.getExpressionTreeString().equals(e2.getExpressionTreeString());
- }
-
- protected static boolean isCompatible(OrderDef o1, OrderDef o2) {
- if (o1 == null && o2 == null) {
- return true;
- }
- if (o1 == null && o2 != null) {
- return false;
- }
- if (o1 != null && o2 == null) {
- return true;
- }
-
- ArrayList<OrderExpressionDef> o1Exprs = o1.getExpressions();
- ArrayList<OrderExpressionDef> o2Exprs = o2.getExpressions();
- if (o1Exprs.size() != o2Exprs.size()) {
- return false;
- }
- for (int i = 0; i < o1Exprs.size(); i++)
- {
- boolean e = isEqual(o1Exprs.get(i), o2Exprs.get(i));
- if (!e) {
- return false;
- }
- }
- return true;
- }
-
- protected static boolean isEqual(OrderExpressionDef e1, OrderExpressionDef e2) {
- boolean e = isEqual((PTFExpressionDef)e1, (PTFExpressionDef)e2);
- if (!e) {
- return false;
- }
- return e1.getOrder().equals(e2.getOrder());
- }
-
- /*
* OI & Serde helper methods
*/
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Mar 20 17:03:54 2013
@@ -10657,22 +10657,22 @@ public class SemanticAnalyzer extends Ba
//--------------------------- Windowing handling: PTFInvocationSpec to PTFDesc --------------------
Operator genWindowingPlan(WindowingSpec wSpec, Operator input) throws SemanticException {
-
+ WindowingComponentizer groups = new WindowingComponentizer(wSpec);
wSpec.fillInWindowingSpecs();
RowResolver rr = opParseCtx.get(input).getRowResolver();
- input = genReduceSinkPlanForWindowing(wSpec, rr, input);
-
- rr = opParseCtx.get(input).getRowResolver();
- PTFTranslator translator = new PTFTranslator();
- PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator);
- /*
- * d. Construct PTF Operator.
- */
- RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
- input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
- new RowSchema(ptfOpRR.getColumnInfos()),
- input), ptfOpRR);
+ while(groups.hasNext() ) {
+ wSpec = groups.next(conf, this, unparseTranslator, rr);
+ input = genReduceSinkPlanForWindowing(wSpec, rr, input);
+ rr = opParseCtx.get(input).getRowResolver();
+ PTFTranslator translator = new PTFTranslator();
+ PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator);
+ RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
+ input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
+ new RowSchema(ptfOpRR.getColumnInfos()),
+ input), ptfOpRR);
+ rr = ptfOpRR;
+ }
dumpOperatorChain(input, null);
Added: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java?rev=1458953&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java (added)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java Wed Mar 20 17:03:54 2013
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
+
+/*
+ * breakup the original WindowingSpec into a set of WindowingSpecs.
+ * Each WindowingSpec is executed in an instance of PTFOperator,
+ * preceded by ReduceSink and Extract.
+ * The logic to componentize is straightforward:
+ * - distribute Window Fn. Specs from original Window Spec into a set of WindowSpecs,
+ * based on their Partitioning.
+ * - A Group of WindowSpecs, is a subset of the Window Fn Invocations in the QueryBlock that
+ * have the same Partitioning(Partition + Order spec).
+ * - Each Group is put in a new WindowingSpec and is evaluated in its own PTFOperator instance.
+ * - the order of computation is then inferred based on the dependency between Groups.
+ * If 2 groups have the same dependency, then the Group with the function that is
+ * earliest in the SelectList is executed first.
+ */
+public class WindowingComponentizer {
+
+ WindowingSpec originalSpec;
+ LinkedHashMap<PartitioningSpec, WindowingSpec> groups;
+
+ public WindowingComponentizer(WindowingSpec originalSpec) throws SemanticException {
+ super();
+ this.originalSpec = originalSpec;
+ groups = new LinkedHashMap<PartitioningSpec, WindowingSpec>();
+ originalSpec.fillInWindowingSpecs();
+ groupFunctions();
+ }
+
+ private void groupFunctions() throws SemanticException {
+ for (WindowExpressionSpec expr : originalSpec.getWindowExpressions()) {
+ WindowFunctionSpec wFn = (WindowFunctionSpec) expr;
+ PartitioningSpec wFnGrp = wFn.getWindowSpec().getPartitioning();
+ WindowingSpec wSpec = groups.get(wFnGrp);
+ if (wSpec == null) {
+ wSpec = new WindowingSpec();
+ groups.put(wFnGrp, wSpec);
+ }
+ wSpec.addWindowFunction(wFn);
+ }
+ }
+
+ public boolean hasNext() {
+ return !groups.isEmpty();
+ }
+
+ public WindowingSpec next(HiveConf hCfg,
+ SemanticAnalyzer semAly,
+ UnparseTranslator unparseT,
+ RowResolver inputRR) throws SemanticException {
+
+ SemanticException originalException = null;
+
+ Iterator<Map.Entry<PartitioningSpec, WindowingSpec>> grpIt = groups.entrySet().iterator();
+ while (grpIt.hasNext()) {
+ Map.Entry<PartitioningSpec, WindowingSpec> entry = grpIt.next();
+ WindowingSpec wSpec = entry.getValue();
+ try {
+ PTFTranslator t = new PTFTranslator();
+ t.translate(wSpec, semAly, hCfg, inputRR, unparseT);
+ groups.remove(entry.getKey());
+ /*
+ * add the filter expression to the last WindowSpec.
+ */
+ if (groups.isEmpty()) {
+ wSpec.setFilterExpr(originalSpec.getFilterExpr());
+ }
+ return wSpec;
+ } catch (SemanticException se) {
+ originalException = se;
+ }
+ }
+
+ throw new SemanticException("Failed to breakup Windowing invocations into Groups. " +
+ "At least 1 group must only depend on input columns. " +
+ "Also check for circular dependencies.\n" +
+ "Underlying error: " + originalException.getMessage());
+ }
+
+}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java Wed Mar 20 17:03:54 2013
@@ -162,6 +162,7 @@ public class WindowingSpec {
fillInWindowSpec(wdwSpec.getSourceId(), wdwSpec, sources);
}
wFn.setWindowSpec(applyContantPartition(wdwSpec));
+ wFn.getWindowSpec().prefixOrderByPartitionSpec();
}
}
}
@@ -387,6 +388,15 @@ public class WindowingSpec {
partitioning = partitioning == null ? new PartitioningSpec() : partitioning;
partitioning.setOrderSpec(orderSpec);
}
+
+ protected void prefixOrderByPartitionSpec() {
+ if ( getOrder() == null ) {
+ setOrder(new OrderSpec());
+ }
+ if ( !getOrder().isPrefixedBy(getPartition()) ) {
+ getOrder().prefixBy(getPartition());
+ }
+ }
};
/*
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q Wed Mar 20 17:03:54 2013
@@ -19,6 +19,4 @@ select p_mfgr,p_name, p_size,
sum(p_size) over (w1) as s ,
dense_rank() as dr
from part
-distribute by p_mfgr
-sort by p_name
-window w1 as (range between p_name 2 preceding and current row);
+window w1 as (partition by p_mfgr order by p_name range between p_name 2 preceding and current row);
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q Wed Mar 20 17:03:54 2013
@@ -18,3 +18,25 @@ create table over10k(
load data local inpath '../data/files/over10k' into table over10k;
select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k limit 100;
+
+select s,
+rank() over (partition by s order by dec desc),
+sum(b) over (partition by s order by ts desc)
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck' ;
+
+select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, rank() over w1,
+rank() over w2
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+window
+w1 as (partition by s order by dec),
+w2 as (partition by si order by f)
+;
\ No newline at end of file
Modified: hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out Wed Mar 20 17:03:54 2013
@@ -33,4 +33,5 @@ PREHOOK: Output: default@part
POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part
POSTHOOK: type: LOAD
POSTHOOK: Output: default@part
-FAILED: SemanticException Primitve type STRING not supported in Value Boundary expression
+FAILED: SemanticException Failed to breakup Windowing invocations into Groups. At least 1 group must only depend on input columns. Also check for circular dependencies.
+Underlying error: Primitve type STRING not supported in Value Boundary expression
Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out Wed Mar 20 17:03:54 2013
@@ -147,3 +147,215 @@ alice hernandez 2 77309413194
alice hernandez 3 77309413194
alice hernandez 4 77309413194
alice hernandez 5 77309413194
+PREHOOK: query: select s,
+rank() over (partition by s order by dec desc),
+sum(b) over (partition by s order by ts desc)
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s,
+rank() over (partition by s order by dec desc),
+sum(b) over (partition by s order by ts desc)
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+bob steinbeck 1 47244642041
+bob steinbeck 11 47244642041
+bob steinbeck 2 47244642041
+bob steinbeck 7 47244642041
+bob steinbeck 8 47244642041
+bob steinbeck 9 47244642041
+bob steinbeck 6 47244642041
+bob steinbeck 10 47244642041
+bob steinbeck 3 47244642041
+bob steinbeck 4 47244642041
+bob steinbeck 5 47244642041
+tom allen 9 81604381169
+tom allen 7 81604381169
+tom allen 16 81604381169
+tom allen 3 81604381169
+tom allen 8 81604381169
+tom allen 10 81604381169
+tom allen 15 81604381169
+tom allen 6 81604381169
+tom allen 18 81604381169
+tom allen 2 81604381169
+tom allen 1 81604381169
+tom allen 5 81604381169
+tom allen 17 81604381169
+tom allen 19 81604381169
+tom allen 11 81604381169
+tom allen 4 81604381169
+tom allen 12 81604381169
+tom allen 13 81604381169
+tom allen 14 81604381169
+PREHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen 1248023 47.16999816894531
+tom allen 1248023 77.77999877929688
+tom allen 1248023 11.069999694824219
+bob steinbeck 722083 83.52999877929688
+bob steinbeck 722083 38.33000183105469
+bob steinbeck 722083 28.479999542236328
+tom allen 1248023 2.8499999046325684
+tom allen 1248023 19.459999084472656
+bob steinbeck 722083 26.290000915527344
+tom allen 1248023 26.239999771118164
+bob steinbeck 722083 36.209999084472656
+tom allen 1248023 95.41000366210938
+tom allen 1248023 89.88999938964844
+tom allen 1248023 14.510000228881836
+bob steinbeck 722083 83.52999877929688
+tom allen 1248023 38.93000030517578
+tom allen 1248023 83.47000122070312
+tom allen 1248023 81.8499984741211
+bob steinbeck 722083 47.810001373291016
+tom allen 1248023 15.84000015258789
+tom allen 1248023 52.779998779296875
+tom allen 1248023 39.4900016784668
+bob steinbeck 722083 80.7300033569336
+tom allen 1248023 11.300000190734863
+tom allen 1248023 68.46999740600586
+bob steinbeck 722083 68.46999740600586
+bob steinbeck 722083 9.699999809265137
+tom allen 1248023 55.38999938964844
+bob steinbeck 722083 132.82000350952148
+tom allen 1248023 132.82000350952148
+PREHOOK: query: select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen 1 1
+tom allen 1 1
+tom allen 7 1
+bob steinbeck 1 1
+bob steinbeck 5 1
+bob steinbeck 5 1
+tom allen 7 1
+tom allen 1 1
+bob steinbeck 5 1
+tom allen 1 1
+bob steinbeck 1 1
+tom allen 7 1
+tom allen 1 1
+tom allen 7 1
+bob steinbeck 5 1
+tom allen 7 1
+tom allen 7 1
+tom allen 7 1
+bob steinbeck 5 1
+tom allen 7 1
+tom allen 7 1
+tom allen 7 1
+bob steinbeck 5 1
+tom allen 7 1
+tom allen 7 1
+bob steinbeck 1 2
+bob steinbeck 5 1
+tom allen 1 1
+bob steinbeck 1 1
+tom allen 7 2
+PREHOOK: query: select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+bob steinbeck 47.810001373291016 1
+tom allen 11.069999694824219 2
+tom allen 54.00000190734863 3
+tom allen 54.00000190734863 4
+tom allen 26.239999771118164 5
+bob steinbeck 28.479999542236328 6
+tom allen 81.8499984741211 7
+tom allen 15.84000015258789 8
+bob steinbeck 83.52999877929688 9
+bob steinbeck 38.33000183105469 10
+bob steinbeck 83.52999877929688 11
+bob steinbeck 80.7300033569336 12
+bob steinbeck 9.699999809265137 13
+tom allen 89.88999938964844 14
+bob steinbeck 26.290000915527344 15
+bob steinbeck 36.209999084472656 16
+tom allen 52.779998779296875 17
+tom allen 11.300000190734863 18
+tom allen 77.77999877929688 19
+tom allen 40.5099983215332 20
+bob steinbeck 82.19000244140625 21
+tom allen 83.47000122070312 22
+tom allen 38.93000030517578 23
+tom allen 47.16999816894531 24
+tom allen 19.459999084472656 25
+tom allen 50.630001068115234 26
+bob steinbeck 27.959999084472656 27
+tom allen 2.8499999046325684 28
+tom allen 95.41000366210938 29
+tom allen 55.38999938964844 30
+PREHOOK: query: select s, rank() over w1,
+rank() over w2
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+window
+w1 as (partition by s order by dec),
+w2 as (partition by si order by f)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, rank() over w1,
+rank() over w2
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+window
+w1 as (partition by s order by dec),
+w2 as (partition by si order by f)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen 14 1
+tom allen 17 1
+tom allen 7 1
+bob steinbeck 1 1
+bob steinbeck 11 1
+bob steinbeck 7 1
+tom allen 12 1
+tom allen 15 1
+bob steinbeck 10 1
+tom allen 13 1
+bob steinbeck 5 1
+tom allen 11 1
+tom allen 2 1
+tom allen 9 1
+bob steinbeck 8 1
+tom allen 3 1
+tom allen 4 1
+tom allen 8 1
+bob steinbeck 3 1
+tom allen 10 1
+tom allen 18 1
+tom allen 19 1
+bob steinbeck 6 1
+tom allen 5 1
+bob steinbeck 9 1
+tom allen 6 2
+bob steinbeck 4 1
+tom allen 16 1
+tom allen 1 1
+bob steinbeck 2 2