You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/03/20 18:03:54 UTC

svn commit: r1458953 - in /hive/branches/ptf-windowing/ql/src: java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/parse/ test/queries/clientnegative/ test/queries/clientpositive/ test/results/clientnegative/ test/results/clientpos...

Author: hashutosh
Date: Wed Mar 20 17:03:54 2013
New Revision: 1458953

URL: http://svn.apache.org/r1458953
Log:
HIVE-4041 [jira] Support multiple partitionings in a single Query
(Harish Butani via Ashutosh Chauhan)

Summary:
add support for multiple partitioning

Currently we disallow queries if the partition specifications of all Wdw fns are not the same. We can relax this by generating multiple PTFOps based on the unique partitionings in a Query. For partitionings that only differ in sort, we can introduce a sort step in between PTFOps, which can happen in the same Reduce task.

Test Plan: added tests to windowing_multipartitioning.q

Reviewers: JIRA, ashutoshc

Reviewed By: ashutoshc

Differential Revision: https://reviews.facebook.net/D9381

Added:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java
Removed:
    hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleDistributeClause.q
    hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q
    hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q
    hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_IncompatibleSortClause.q
    hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleDistributeClause.q.out
    hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleOrderInWindowDefs.q.out
    hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatiblePartitionInWindowDefs.q.out
    hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_IncompatibleSortClause.q.out
Modified:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
    hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
    hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q
    hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out
    hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Wed Mar 20 17:03:54 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -208,12 +209,12 @@ public final class ColumnPrunerProcFacto
     private static RowResolver buildPrunedRR(List<String> prunedCols,
         RowResolver oldRR, ArrayList<ColumnInfo> sig) throws SemanticException{
       RowResolver newRR = new RowResolver();
-      for (String col : prunedCols) {
-        String[] nm = oldRR.reverseLookup(col);
-        ColumnInfo colInfo = oldRR.get(nm[0], nm[1]);
-        if (colInfo != null) {
-          newRR.put(nm[0], nm[1], colInfo);
-          sig.add(colInfo);
+      HashSet<String> prunedColsSet = new HashSet<String>(prunedCols);
+      for(ColumnInfo cInfo : oldRR.getRowSchema().getSignature()) {
+        if ( prunedColsSet.contains(cInfo.getInternalName())) {
+          String[] nm = oldRR.reverseLookup(cInfo.getInternalName());
+          newRR.put(nm[0], nm[1], cInfo);
+          sig.add(cInfo);
         }
       }
       return newRR;

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java Wed Mar 20 17:03:54 2013
@@ -339,7 +339,7 @@ public class PTFInvocationSpec {
     public int hashCode() {
       final int prime = 31;
       int result = 1;
-      result = prime * result + ((expression == null) ? 0 : expression.hashCode());
+      result = prime * result + ((expression == null) ? 0 : expression.toStringTree().hashCode());
       return result;
     }
 
@@ -351,7 +351,7 @@ public class PTFInvocationSpec {
       if (obj == null) {
         return false;
       }
-      if (getClass() != obj.getClass()) {
+      if (!getClass().isAssignableFrom(obj.getClass())) {
         return false;
       }
       PartitionExpression other = (PartitionExpression) obj;
@@ -408,6 +408,38 @@ public class PTFInvocationSpec {
       expressions.add(c);
     }
 
+    protected boolean isPrefixedBy(PartitionSpec pSpec) {
+      if ( pSpec == null || pSpec.getExpressions() == null) {
+        return true;
+      }
+
+      int pExprCnt = pSpec.getExpressions().size();
+      int exprCnt = getExpressions() == null ? 0 : getExpressions().size();
+
+      if ( exprCnt < pExprCnt ) {
+        return false;
+      }
+
+      for(int i=0; i < pExprCnt; i++) {
+        if ( !pSpec.getExpressions().get(i).equals(getExpressions().get(i)) ) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    protected void prefixBy(PartitionSpec pSpec) {
+      if ( pSpec == null || pSpec.getExpressions() == null) {
+        return;
+      }
+      if ( expressions == null ) {
+        expressions = new ArrayList<PTFInvocationSpec.OrderExpression>();
+      }
+      for(int i = pSpec.getExpressions().size() - 1; i >= 0; i--) {
+        expressions.add(0, new OrderExpression(pSpec.getExpressions().get(i)));
+      }
+    }
+
     @Override
     public int hashCode()
     {

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Wed Mar 20 17:03:54 2013
@@ -477,6 +477,14 @@ public class PTFTranslator {
       return;
     }
     PartitionDef partDef = translate(def.getRawInputShape(), spec.getPartition());
+
+    if ( spec.getOrder() == null ) {
+      spec.setOrder(new OrderSpec());
+    }
+    if ( !spec.getOrder().isPrefixedBy(spec.getPartition()) ) {
+      spec.getOrder().prefixBy(spec.getPartition());
+    }
+
     OrderDef orderDef = translate(def.getRawInputShape(), spec.getOrder(), partDef);
     def.setPartition(partDef);
     def.setOrder(orderDef);
@@ -535,19 +543,6 @@ public class PTFTranslator {
       OrderSpec spec,
       PartitionDef partitionDef) throws SemanticException {
 
-    if (spec == null || spec.getExpressions() == null || spec.getExpressions().size() == 0)
-    {
-      if (partitionDef == null) {
-        return null;
-      }
-      return new OrderDef(partitionDef);
-    }
-
-    if (partitionDef == null) {
-      throw new SemanticException(String.format(
-          "Cannot have an Order spec (%s) w/o a Partition spec", spec));
-    }
-
     OrderDef def = new OrderDef();
     for (OrderExpression oExpr : spec.getExpressions())
     {
@@ -555,46 +550,6 @@ public class PTFTranslator {
       def.addExpression(oexpDef);
     }
 
-    /*
-     * either all partition expressions must be in Order list or none must be specified.
-     * If none are specified then add them all.
-     */
-    int numOfPartColumns = 0;
-    List<OrderExpressionDef> orderExprs = def.getExpressions();
-    List<PTFExpressionDef> partExprs = partitionDef.getExpressions();
-    int chkSize = partExprs.size();
-    chkSize = chkSize > orderExprs.size() ? orderExprs.size() : chkSize;
-    for (int i = 0; i < chkSize; i++)
-    {
-      if (orderExprs.get(i).getExpressionTreeString()
-          .equals(partExprs.get(i).getExpressionTreeString()))
-      {
-        numOfPartColumns++;
-      } else {
-        break;
-      }
-    }
-
-    if (numOfPartColumns != 0 && numOfPartColumns != partExprs.size())
-    {
-      throw new SemanticException(
-          String.format(
-                  "all partition columns must be in order clause(%s) or none should be specified",
-                  spec));
-    }
-
-    if (numOfPartColumns == 0)
-    {
-      ArrayList<OrderExpressionDef> combinedOrderCols = new ArrayList<OrderExpressionDef>();
-      for (PTFExpressionDef eDef : partExprs)
-      {
-        OrderExpressionDef oeDef = new OrderExpressionDef(eDef);
-        combinedOrderCols.add(oeDef);
-      }
-      combinedOrderCols.addAll(orderExprs);
-      def.setExpressions(combinedOrderCols);
-    }
-
     return def;
   }
 
@@ -620,12 +575,12 @@ public class PTFTranslator {
     return oexpDef;
   }
 
-  private WindowFrameDef translate(String wFnName, ShapeDetails inpShape, WindowSpec spec) throws SemanticException {
-    PartitionSpec pSpec = spec.getPartition();
-    OrderSpec oSpec = spec.getOrder();
-    PartitionDef pDef = translate(inpShape, pSpec);
-    OrderDef oDef = translate(inpShape, oSpec, pDef);
-    validatePartitioningForWdwFn(wFnName, pDef, oDef);
+  private WindowFrameDef translate(String wFnName, ShapeDetails inpShape, WindowSpec spec)
+      throws SemanticException {
+    /*
+     * Since we componentize Windowing, no need to translate
+     * the Partition & Order specs of individual WFns.
+    */
     return translate(inpShape, spec.getWindowFrame());
   }
 
@@ -740,25 +695,6 @@ public class PTFTranslator {
 
   }
 
-  private void validatePartitioningForWdwFn(String wFnName, PartitionDef fPart, OrderDef fOrder)
-      throws SemanticException
-  {
-    PartitionDef tPart = ptfDesc.getFuncDef().getPartition();
-
-    if (!PTFTranslator.isCompatible(tPart, fPart))
-    {
-      throw new SemanticException(
-          String.format("Window Function '%s' has an incompatible partition clause", wFnName));
-    }
-
-    OrderDef tOrder = ptfDesc.getFuncDef().getOrder();
-    if (!PTFTranslator.isCompatible(tOrder, fOrder))
-    {
-      throw new SemanticException(
-          String.format("Window Function '%s' has an incompatible order clause", wFnName));
-    }
-  }
-
   private ShapeDetails setupTableFnShape(String fnName, ShapeDetails inpShape, StructObjectInspector OI, ArrayList<String> columnNames, RowResolver rr)
       throws SemanticException {
     if (fnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION)
@@ -956,84 +892,6 @@ public class PTFTranslator {
   }
 
   /*
-   * Partition & Order expression compatibility methods
-   */
-
-  protected static boolean isCompatible(PartitionDef p1, PartitionDef p2) {
-    if (p1 == null && p2 == null) {
-      return true;
-    }
-    if (p1 == null && p2 != null) {
-      return false;
-    }
-    if (p1 != null && p2 == null) {
-      return true;
-    }
-
-    ArrayList<PTFExpressionDef> p1Exprs = p1.getExpressions();
-    ArrayList<PTFExpressionDef> p2Exprs = p2.getExpressions();
-    if (p1Exprs.size() != p2Exprs.size()) {
-      return false;
-    }
-    for (int i = 0; i < p1Exprs.size(); i++)
-    {
-      boolean e = isEqual(p1Exprs.get(i), p2Exprs.get(i));
-      if (!e) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  protected static boolean isEqual(PTFExpressionDef e1, PTFExpressionDef e2) {
-    if (e1 == null && e2 == null) {
-      return false;
-    }
-    if (e1 == null && e2 != null) {
-      return false;
-    }
-    if (e1 != null && e2 == null) {
-      return false;
-    }
-
-    return e1.getExpressionTreeString().equals(e2.getExpressionTreeString());
-  }
-
-  protected static boolean isCompatible(OrderDef o1, OrderDef o2) {
-    if (o1 == null && o2 == null) {
-      return true;
-    }
-    if (o1 == null && o2 != null) {
-      return false;
-    }
-    if (o1 != null && o2 == null) {
-      return true;
-    }
-
-    ArrayList<OrderExpressionDef> o1Exprs = o1.getExpressions();
-    ArrayList<OrderExpressionDef> o2Exprs = o2.getExpressions();
-    if (o1Exprs.size() != o2Exprs.size()) {
-      return false;
-    }
-    for (int i = 0; i < o1Exprs.size(); i++)
-    {
-      boolean e = isEqual(o1Exprs.get(i), o2Exprs.get(i));
-      if (!e) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  protected static boolean isEqual(OrderExpressionDef e1, OrderExpressionDef e2) {
-    boolean e = isEqual((PTFExpressionDef)e1, (PTFExpressionDef)e2);
-    if (!e) {
-      return false;
-    }
-    return e1.getOrder().equals(e2.getOrder());
-  }
-
-  /*
    * OI & Serde helper methods
    */
 

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Mar 20 17:03:54 2013
@@ -10657,22 +10657,22 @@ public class SemanticAnalyzer extends Ba
 //--------------------------- Windowing handling: PTFInvocationSpec to PTFDesc --------------------
 
   Operator genWindowingPlan(WindowingSpec wSpec, Operator input) throws SemanticException {
-
+    WindowingComponentizer groups = new WindowingComponentizer(wSpec);
     wSpec.fillInWindowingSpecs();
     RowResolver rr = opParseCtx.get(input).getRowResolver();
-    input = genReduceSinkPlanForWindowing(wSpec, rr, input);
-
-    rr = opParseCtx.get(input).getRowResolver();
-    PTFTranslator translator = new PTFTranslator();
-    PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator);
 
-    /*
-     * d. Construct PTF Operator.
-     */
-    RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
-    input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
-        new RowSchema(ptfOpRR.getColumnInfos()),
-        input), ptfOpRR);
+    while(groups.hasNext() ) {
+      wSpec = groups.next(conf, this, unparseTranslator, rr);
+      input = genReduceSinkPlanForWindowing(wSpec, rr, input);
+      rr = opParseCtx.get(input).getRowResolver();
+      PTFTranslator translator = new PTFTranslator();
+      PTFDesc ptfDesc = translator.translate(wSpec, this, conf, rr, unparseTranslator);
+      RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
+      input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc,
+          new RowSchema(ptfOpRR.getColumnInfos()),
+          input), ptfOpRR);
+      rr = ptfOpRR;
+    }
 
     dumpOperatorChain(input, null);
 

Added: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java?rev=1458953&view=auto
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java (added)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingComponentizer.java Wed Mar 20 17:03:54 2013
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
+
+/*
+ * breakup the original WindowingSpec into a set of WindowingSpecs.
+ * Each WindowingSpec is executed in an instance of PTFOperator,
+ * preceded by ReduceSink and Extract.
+ * The logic to componentize is straightforward:
+ * - distribute Window Fn. Specs from original Window Spec into a set of WindowSpecs,
+ *   based on their Partitioning.
+ * - A Group of WindowSpecs, is a subset of the Window Fn Invocations in the QueryBlock that
+ *   have the same Partitioning(Partition + Order spec).
+ * - Each Group is put in a new WindowingSpec and is evaluated in its own PTFOperator instance.
+ * - the order of computation is then inferred based on the dependency between Groups.
+ *   If 2 groups have the same dependency, then the Group with the function that is
+ *   earliest in the SelectList is executed first.
+ */
+public class WindowingComponentizer {
+
+  WindowingSpec originalSpec;
+  LinkedHashMap<PartitioningSpec, WindowingSpec> groups;
+
+  public WindowingComponentizer(WindowingSpec originalSpec) throws SemanticException {
+    super();
+    this.originalSpec = originalSpec;
+    groups = new LinkedHashMap<PartitioningSpec, WindowingSpec>();
+    originalSpec.fillInWindowingSpecs();
+    groupFunctions();
+  }
+
+  private void groupFunctions() throws SemanticException {
+    for (WindowExpressionSpec expr : originalSpec.getWindowExpressions()) {
+      WindowFunctionSpec wFn = (WindowFunctionSpec) expr;
+      PartitioningSpec wFnGrp = wFn.getWindowSpec().getPartitioning();
+      WindowingSpec wSpec = groups.get(wFnGrp);
+      if (wSpec == null) {
+        wSpec = new WindowingSpec();
+        groups.put(wFnGrp, wSpec);
+      }
+      wSpec.addWindowFunction(wFn);
+    }
+  }
+
+  public boolean hasNext() {
+    return !groups.isEmpty();
+  }
+
+  public WindowingSpec next(HiveConf hCfg,
+      SemanticAnalyzer semAly,
+      UnparseTranslator unparseT,
+      RowResolver inputRR) throws SemanticException {
+
+    SemanticException originalException = null;
+
+    Iterator<Map.Entry<PartitioningSpec, WindowingSpec>> grpIt = groups.entrySet().iterator();
+    while (grpIt.hasNext()) {
+      Map.Entry<PartitioningSpec, WindowingSpec> entry = grpIt.next();
+      WindowingSpec wSpec = entry.getValue();
+      try {
+        PTFTranslator t = new PTFTranslator();
+        t.translate(wSpec, semAly, hCfg, inputRR, unparseT);
+        groups.remove(entry.getKey());
+        /*
+         * add the filter expression to the last WindowSpec.
+         */
+        if (groups.isEmpty()) {
+          wSpec.setFilterExpr(originalSpec.getFilterExpr());
+        }
+        return wSpec;
+      } catch (SemanticException se) {
+        originalException = se;
+      }
+    }
+
+    throw new SemanticException("Failed to breakup Windowing invocations into Groups. " +
+        "At least 1 group must only depend on input columns. " +
+        "Also check for circular dependencies.\n" +
+        "Underlying error: " + originalException.getMessage());
+  }
+
+}

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java Wed Mar 20 17:03:54 2013
@@ -162,6 +162,7 @@ public class WindowingSpec {
             fillInWindowSpec(wdwSpec.getSourceId(), wdwSpec, sources);
           }
           wFn.setWindowSpec(applyContantPartition(wdwSpec));
+          wFn.getWindowSpec().prefixOrderByPartitionSpec();
         }
       }
     }
@@ -387,6 +388,15 @@ public class WindowingSpec {
       partitioning = partitioning == null ? new PartitioningSpec() : partitioning;
       partitioning.setOrderSpec(orderSpec);
     }
+
+    protected void prefixOrderByPartitionSpec() {
+      if ( getOrder() == null ) {
+        setOrder(new OrderSpec());
+      }
+      if ( !getOrder().isPrefixedBy(getPartition()) ) {
+        getOrder().prefixBy(getPartition());
+      }
+    }
   };
 
   /*

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q Wed Mar 20 17:03:54 2013
@@ -19,6 +19,4 @@ select  p_mfgr,p_name, p_size,   
 sum(p_size) over (w1) as s ,    
 dense_rank() as dr  
 from part  
-distribute by p_mfgr  
-sort by p_name  
-window w1 as (range between p_name 2 preceding and current row);
+window w1 as (partition by p_mfgr order by p_name range between p_name 2 preceding and current row);

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_multipartitioning.q Wed Mar 20 17:03:54 2013
@@ -18,3 +18,25 @@ create table over10k(
 load data local inpath '../data/files/over10k' into table over10k;
 
 select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k limit 100;
+
+select s, 
+rank() over (partition by s order by dec desc), 
+sum(b) over (partition by s order by ts desc) 
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck' ;
+
+select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck';
+
+select s, rank() over w1, 
+rank() over w2 
+from over10k 
+where s = 'tom allen' or s = 'bob steinbeck'
+window 
+w1 as (partition by s order by dec), 
+w2 as (partition by si order by f) 
+;
\ No newline at end of file

Modified: hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientnegative/ptf_negative_InvalidValueBoundary.q.out Wed Mar 20 17:03:54 2013
@@ -33,4 +33,5 @@ PREHOOK: Output: default@part
 POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part
 POSTHOOK: type: LOAD
 POSTHOOK: Output: default@part
-FAILED: SemanticException Primitve type STRING not supported in Value Boundary expression
+FAILED: SemanticException Failed to breakup Windowing invocations into Groups. At least 1 group must only depend on input columns. Also check for circular dependencies.
+Underlying error: Primitve type STRING not supported in Value Boundary expression

Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out?rev=1458953&r1=1458952&r2=1458953&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_multipartitioning.q.out Wed Mar 20 17:03:54 2013
@@ -147,3 +147,215 @@ alice hernandez	2	77309413194
 alice hernandez	3	77309413194
 alice hernandez	4	77309413194
 alice hernandez	5	77309413194
+PREHOOK: query: select s, 
+rank() over (partition by s order by dec desc), 
+sum(b) over (partition by s order by ts desc) 
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, 
+rank() over (partition by s order by dec desc), 
+sum(b) over (partition by s order by ts desc) 
+from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+bob steinbeck	1	47244642041
+bob steinbeck	11	47244642041
+bob steinbeck	2	47244642041
+bob steinbeck	7	47244642041
+bob steinbeck	8	47244642041
+bob steinbeck	9	47244642041
+bob steinbeck	6	47244642041
+bob steinbeck	10	47244642041
+bob steinbeck	3	47244642041
+bob steinbeck	4	47244642041
+bob steinbeck	5	47244642041
+tom allen	9	81604381169
+tom allen	7	81604381169
+tom allen	16	81604381169
+tom allen	3	81604381169
+tom allen	8	81604381169
+tom allen	10	81604381169
+tom allen	15	81604381169
+tom allen	6	81604381169
+tom allen	18	81604381169
+tom allen	2	81604381169
+tom allen	1	81604381169
+tom allen	5	81604381169
+tom allen	17	81604381169
+tom allen	19	81604381169
+tom allen	11	81604381169
+tom allen	4	81604381169
+tom allen	12	81604381169
+tom allen	13	81604381169
+tom allen	14	81604381169
+PREHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen	1248023	47.16999816894531
+tom allen	1248023	77.77999877929688
+tom allen	1248023	11.069999694824219
+bob steinbeck	722083	83.52999877929688
+bob steinbeck	722083	38.33000183105469
+bob steinbeck	722083	28.479999542236328
+tom allen	1248023	2.8499999046325684
+tom allen	1248023	19.459999084472656
+bob steinbeck	722083	26.290000915527344
+tom allen	1248023	26.239999771118164
+bob steinbeck	722083	36.209999084472656
+tom allen	1248023	95.41000366210938
+tom allen	1248023	89.88999938964844
+tom allen	1248023	14.510000228881836
+bob steinbeck	722083	83.52999877929688
+tom allen	1248023	38.93000030517578
+tom allen	1248023	83.47000122070312
+tom allen	1248023	81.8499984741211
+bob steinbeck	722083	47.810001373291016
+tom allen	1248023	15.84000015258789
+tom allen	1248023	52.779998779296875
+tom allen	1248023	39.4900016784668
+bob steinbeck	722083	80.7300033569336
+tom allen	1248023	11.300000190734863
+tom allen	1248023	68.46999740600586
+bob steinbeck	722083	68.46999740600586
+bob steinbeck	722083	9.699999809265137
+tom allen	1248023	55.38999938964844
+bob steinbeck	722083	132.82000350952148
+tom allen	1248023	132.82000350952148
+PREHOOK: query: select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k
+where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen	1	1
+tom allen	1	1
+tom allen	7	1
+bob steinbeck	1	1
+bob steinbeck	5	1
+bob steinbeck	5	1
+tom allen	7	1
+tom allen	1	1
+bob steinbeck	5	1
+tom allen	1	1
+bob steinbeck	1	1
+tom allen	7	1
+tom allen	1	1
+tom allen	7	1
+bob steinbeck	5	1
+tom allen	7	1
+tom allen	7	1
+tom allen	7	1
+bob steinbeck	5	1
+tom allen	7	1
+tom allen	7	1
+tom allen	7	1
+bob steinbeck	5	1
+tom allen	7	1
+tom allen	7	1
+bob steinbeck	1	2
+bob steinbeck	5	1
+tom allen	1	1
+bob steinbeck	1	1
+tom allen	7	2
+PREHOOK: query: select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, sum(f) over (partition by i), row_number() over () from over10k where s = 'tom allen' or s = 'bob steinbeck'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+bob steinbeck	47.810001373291016	1
+tom allen	11.069999694824219	2
+tom allen	54.00000190734863	3
+tom allen	54.00000190734863	4
+tom allen	26.239999771118164	5
+bob steinbeck	28.479999542236328	6
+tom allen	81.8499984741211	7
+tom allen	15.84000015258789	8
+bob steinbeck	83.52999877929688	9
+bob steinbeck	38.33000183105469	10
+bob steinbeck	83.52999877929688	11
+bob steinbeck	80.7300033569336	12
+bob steinbeck	9.699999809265137	13
+tom allen	89.88999938964844	14
+bob steinbeck	26.290000915527344	15
+bob steinbeck	36.209999084472656	16
+tom allen	52.779998779296875	17
+tom allen	11.300000190734863	18
+tom allen	77.77999877929688	19
+tom allen	40.5099983215332	20
+bob steinbeck	82.19000244140625	21
+tom allen	83.47000122070312	22
+tom allen	38.93000030517578	23
+tom allen	47.16999816894531	24
+tom allen	19.459999084472656	25
+tom allen	50.630001068115234	26
+bob steinbeck	27.959999084472656	27
+tom allen	2.8499999046325684	28
+tom allen	95.41000366210938	29
+tom allen	55.38999938964844	30
+PREHOOK: query: select s, rank() over w1, 
+rank() over w2 
+from over10k 
+where s = 'tom allen' or s = 'bob steinbeck'
+window 
+w1 as (partition by s order by dec), 
+w2 as (partition by si order by f)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, rank() over w1, 
+rank() over w2 
+from over10k 
+where s = 'tom allen' or s = 'bob steinbeck'
+window 
+w1 as (partition by s order by dec), 
+w2 as (partition by si order by f)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+tom allen	14	1
+tom allen	17	1
+tom allen	7	1
+bob steinbeck	1	1
+bob steinbeck	11	1
+bob steinbeck	7	1
+tom allen	12	1
+tom allen	15	1
+bob steinbeck	10	1
+tom allen	13	1
+bob steinbeck	5	1
+tom allen	11	1
+tom allen	2	1
+tom allen	9	1
+bob steinbeck	8	1
+tom allen	3	1
+tom allen	4	1
+tom allen	8	1
+bob steinbeck	3	1
+tom allen	10	1
+tom allen	18	1
+tom allen	19	1
+bob steinbeck	6	1
+tom allen	5	1
+bob steinbeck	9	1
+tom allen	6	2
+bob steinbeck	4	1
+tom allen	16	1
+tom allen	1	1
+bob steinbeck	2	2