You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2016/03/16 20:03:34 UTC
hive git commit: HIVE-4662: first_value can't have more than one
order by column (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 06b604a03 -> 0b574501c
HIVE-4662: first_value can't have more than one order by column (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0b574501
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0b574501
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0b574501
Branch: refs/heads/master
Commit: 0b574501c538d8011898fbe23a712feffc2925b6
Parents: 06b604a
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Fri Mar 11 08:14:28 2016 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed Mar 16 20:02:55 2016 +0100
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/PTFTranslator.java | 25 +-
.../hadoop/hive/ql/parse/WindowingSpec.java | 56 +-
.../hadoop/hive/ql/plan/PTFDeserializer.java | 5 +-
.../hive/ql/plan/ptf/PTFExpressionDef.java | 3 +-
.../hive/ql/plan/ptf/ValueBoundaryDef.java | 16 +-
.../hive/ql/udf/ptf/WindowingTableFunction.java | 303 ++++--
.../clientpositive/windowing_range_multiorder.q | 34 +
.../windowing_range_multiorder.q.out | 910 +++++++++++++++++++
8 files changed, 1232 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
index 9921b21..018d8d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
@@ -31,8 +31,6 @@ import java.util.Stack;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.TreeWizard;
import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -99,6 +97,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PTFTranslator {
@@ -547,15 +547,20 @@ public class PTFTranslator {
if (bndSpec instanceof ValueBoundarySpec) {
ValueBoundarySpec vBndSpec = (ValueBoundarySpec) bndSpec;
ValueBoundaryDef vbDef = new ValueBoundaryDef(vBndSpec.getDirection(), vBndSpec.getAmt());
- PTFTranslator.validateNoLeadLagInValueBoundarySpec(vBndSpec.getExpression());
- PTFExpressionDef exprDef = null;
- try {
- exprDef = buildExpressionDef(inpShape, vBndSpec.getExpression());
- } catch (HiveException he) {
- throw new SemanticException(he);
+ for (OrderExpression oe : vBndSpec.getOrderExpressions()) {
+ PTFTranslator.validateNoLeadLagInValueBoundarySpec(oe.getExpression());
+ PTFExpressionDef exprDef = null;
+ try {
+ exprDef = buildExpressionDef(inpShape, oe.getExpression());
+ } catch (HiveException he) {
+ throw new SemanticException(he);
+ }
+ PTFTranslator.validateValueBoundaryExprType(exprDef.getOI());
+ OrderExpressionDef orderExprDef = new OrderExpressionDef(exprDef);
+ orderExprDef.setOrder(oe.getOrder());
+ orderExprDef.setNullOrder(oe.getNullOrder());
+ vbDef.addOrderExpressionDef(orderExprDef);
}
- PTFTranslator.validateValueBoundaryExprType(exprDef.getOI());
- vbDef.setExpressionDef(exprDef);
return vbDef;
}
else if (bndSpec instanceof RangeBoundarySpec) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
index 1bfe8d9..5ce7200 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.hive.ql.parse;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import org.antlr.runtime.CommonToken;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec;
@@ -295,17 +297,30 @@ public class WindowingSpec {
BoundarySpec end = wFrame.getEnd();
if (start instanceof ValueBoundarySpec || end instanceof ValueBoundarySpec) {
- if ( order != null ) {
- if ( order.getExpressions().size() > 1 ) {
- throw new SemanticException("Range based Window Frame can have only 1 Sort Key");
- }
+ if (order == null || order.getExpressions().size() == 0) {
+ throw new SemanticException("Range based Window Frame needs to specify ORDER BY clause");
+ }
- if (start instanceof ValueBoundarySpec) {
- ((ValueBoundarySpec)start).setExpression(order.getExpressions().get(0).getExpression());
- }
- if (end instanceof ValueBoundarySpec) {
- ((ValueBoundarySpec)end).setExpression(order.getExpressions().get(0).getExpression());
- }
+ boolean defaultPreceding = start.getDirection() == Direction.PRECEDING &&
+ start.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT &&
+ end.getDirection() == Direction.CURRENT;
+ boolean defaultFollowing = start.getDirection() == Direction.CURRENT &&
+ end.getDirection() == Direction.FOLLOWING &&
+ end.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT;
+ boolean defaultPrecedingFollowing = start.getDirection() == Direction.PRECEDING &&
+ start.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT &&
+ end.getDirection() == Direction.FOLLOWING &&
+ end.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT;
+ boolean multiOrderAllowed = defaultPreceding || defaultFollowing || defaultPrecedingFollowing;
+ if ( order.getExpressions().size() != 1 && !multiOrderAllowed) {
+ throw new SemanticException("Range value based Window Frame can have only 1 Sort Key");
+ }
+
+ if (start instanceof ValueBoundarySpec) {
+ ((ValueBoundarySpec)start).setOrderExpressions(order.getExpressions());
+ }
+ if (end instanceof ValueBoundarySpec) {
+ ((ValueBoundarySpec)end).setOrderExpressions(order.getExpressions());
}
}
}
@@ -683,8 +698,8 @@ public class WindowingSpec {
public static class ValueBoundarySpec extends BoundarySpec
{
Direction direction;
- ASTNode expression;
int amt;
+ List<OrderExpression> orderExpressions;
public ValueBoundarySpec() {
}
@@ -708,14 +723,14 @@ public class WindowingSpec {
this.direction = direction;
}
- public ASTNode getExpression()
+ public List<OrderExpression> getOrderExpressions()
{
- return expression;
+ return orderExpressions;
}
- public void setExpression(ASTNode expression)
+ public void setOrderExpressions(List<OrderExpression> orderExpressions)
{
- this.expression = expression;
+ this.orderExpressions = orderExpressions;
}
@Override
@@ -733,7 +748,16 @@ public class WindowingSpec {
@Override
public String toString()
{
- return String.format("value(%s %s %s)", expression.toStringTree(), amt, direction);
+ StringBuilder exprs = new StringBuilder();
+ if (orderExpressions != null) {
+ for (int i=0; i<orderExpressions.size(); i++) {
+ exprs.append(i == 0 ? orderExpressions.get(i).getExpression().toStringTree()
+ : ", " + orderExpressions.get(i).getExpression().toStringTree());
+ }
+ } else {
+ exprs.append("No order expression");
+ }
+ return String.format("value(%s %s %s)", exprs.toString(), amt, direction);
}
public int compareTo(BoundarySpec other)
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
index 830a8eb..cfddb22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.LeadLagInfo;
import org.apache.hadoop.hive.ql.parse.WindowingExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFQueryInputDef;
@@ -214,7 +215,9 @@ public class PTFDeserializer {
protected void initialize(BoundaryDef def, ShapeDetails inpShape) throws HiveException {
if (def instanceof ValueBoundaryDef) {
ValueBoundaryDef vDef = (ValueBoundaryDef) def;
- initialize(vDef.getExpressionDef(), inpShape);
+ for (OrderExpressionDef exprDef : vDef.getOrderDef().getExpressions()) {
+ initialize(exprDef, inpShape);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java
index fa7fc76..aa82fbf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java
@@ -19,10 +19,9 @@
package org.apache.hadoop.hive.ql.plan.ptf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
-import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.plan.Explain;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class PTFExpressionDef {
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
index 3725ac8..e1a8a90 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
@@ -19,16 +19,16 @@
package org.apache.hadoop.hive.ql.plan.ptf;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class ValueBoundaryDef extends BoundaryDef {
- private PTFExpressionDef expressionDef;
+ private OrderDef orderDef;
private final int amt;
private final int relativeOffset;
public ValueBoundaryDef(Direction direction, int amt) {
this.direction = direction;
this.amt = amt;
+ this.orderDef = new OrderDef();
// Calculate relative offset
switch(this.direction) {
@@ -52,16 +52,12 @@ public class ValueBoundaryDef extends BoundaryDef {
return this.direction == Direction.PRECEDING ? vb.amt - this.amt : this.amt - vb.amt;
}
- public PTFExpressionDef getExpressionDef() {
- return expressionDef;
+ public OrderDef getOrderDef() {
+ return orderDef;
}
- public void setExpressionDef(PTFExpressionDef expressionDef) {
- this.expressionDef = expressionDef;
- }
-
- public ObjectInspector getOI() {
- return expressionDef == null ? null : expressionDef.getOI();
+ public void addOrderExpressionDef(OrderExpressionDef expressionDef) {
+ this.orderDef.addExpression(expressionDef);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
index 2ac4039..858b47a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
@@ -38,13 +38,14 @@ import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator;
import org.apache.hadoop.hive.ql.exec.PTFRollingPartition;
import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.OrderDef;
+import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef;
@@ -110,9 +111,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
StructObjectInspector inputOI = iPart.getOutputOI();
WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
- Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder();
- NullOrder nullOrder = wTFnDef.getOrder().getExpressions().get(0).getNullOrder();
-
for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) {
boolean processWindow = processWindow(wFn);
pItr.reset();
@@ -123,7 +121,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
oColumns.add((List<?>)out);
} else {
- oColumns.add(executeFnwithWindow(getQueryDef(), wFn, iPart, order, nullOrder));
+ oColumns.add(executeFnwithWindow(getQueryDef(), wFn, iPart));
}
}
@@ -422,8 +420,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
} else {
int rowToProcess = streamingState.rollingPart.rowToProcess(wFn.getWindowFrame());
if (rowToProcess >= 0) {
- Range rng = getRange(wFn, rowToProcess, streamingState.rollingPart,
- streamingState.order, streamingState.nullOrder);
+ Range rng = getRange(wFn, rowToProcess, streamingState.rollingPart);
PTFPartitionIterator<Object> rItr = rng.iterator();
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, rItr);
Object out = evaluateWindowFunction(wFn, rItr);
@@ -500,8 +497,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
while (numRowsRemaining > 0) {
int rowToProcess = streamingState.rollingPart.size() - numRowsRemaining;
if (rowToProcess >= 0) {
- Range rng = getRange(wFn, rowToProcess, streamingState.rollingPart,
- streamingState.order, streamingState.nullOrder);
+ Range rng = getRange(wFn, rowToProcess, streamingState.rollingPart);
PTFPartitionIterator<Object> rItr = rng.iterator();
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, rItr);
Object out = evaluateWindowFunction(wFn, rItr);
@@ -660,13 +656,11 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
ArrayList<Object> executeFnwithWindow(PTFDesc ptfDesc,
WindowFunctionDef wFnDef,
- PTFPartition iPart,
- Order order,
- NullOrder nullOrder)
+ PTFPartition iPart)
throws HiveException {
ArrayList<Object> vals = new ArrayList<Object>();
for(int i=0; i < iPart.size(); i++) {
- Range rng = getRange(wFnDef, i, iPart, order, nullOrder);
+ Range rng = getRange(wFnDef, i, iPart);
PTFPartitionIterator<Object> rItr = rng.iterator();
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, rItr);
Object out = evaluateWindowFunction(wFnDef, rItr);
@@ -675,7 +669,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
return vals;
}
- private Range getRange(WindowFunctionDef wFnDef, int currRow, PTFPartition p, Order order, NullOrder nullOrder) throws HiveException
+ private Range getRange(WindowFunctionDef wFnDef, int currRow, PTFPartition p) throws HiveException
{
BoundaryDef startB = wFnDef.getWindowFrame().getStart();
BoundaryDef endB = wFnDef.getWindowFrame().getEnd();
@@ -686,18 +680,26 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
int start, end;
-
if (rowFrame) {
start = getRowBoundaryStart(startB, currRow);
end = getRowBoundaryEnd(endB, currRow, p);
- }
- else {
+ } else {
ValueBoundaryScanner vbs;
if ( startB instanceof ValueBoundaryDef ) {
- vbs = ValueBoundaryScanner.getScanner((ValueBoundaryDef)startB, order, nullOrder);
+ ValueBoundaryDef startValueBoundaryDef = (ValueBoundaryDef)startB;
+ if (startValueBoundaryDef.getOrderDef().getExpressions().size() != 1) {
+ vbs = MultiValueBoundaryScanner.getScanner(startValueBoundaryDef);
+ } else {
+ vbs = SingleValueBoundaryScanner.getScanner(startValueBoundaryDef);
+ }
}
else {
- vbs = ValueBoundaryScanner.getScanner((ValueBoundaryDef)endB, order, nullOrder);
+ ValueBoundaryDef endValueBoundaryDef = (ValueBoundaryDef)endB;
+ if (endValueBoundaryDef.getOrderDef().getExpressions().size() != 1) {
+ vbs = MultiValueBoundaryScanner.getScanner(endValueBoundaryDef);
+ } else {
+ vbs = SingleValueBoundaryScanner.getScanner(endValueBoundaryDef);
+ }
}
vbs.reset(startB);
start = vbs.computeStart(currRow, p);
@@ -770,29 +772,35 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- /*
- * - starting from the given rowIdx scan in the given direction until a row's expr
- * evaluates to an amt that crosses the 'amt' threshold specified in the ValueBoundaryDef.
- */
- static abstract class ValueBoundaryScanner
- {
+
+ static abstract class ValueBoundaryScanner {
BoundaryDef bndDef;
- Order order;
- NullOrder nullOrder;
- PTFExpressionDef expressionDef;
- public ValueBoundaryScanner(BoundaryDef bndDef, Order order, NullOrder nullOrder, PTFExpressionDef expressionDef)
- {
+ public ValueBoundaryScanner(BoundaryDef bndDef) {
this.bndDef = bndDef;
- this.order = order;
- this.nullOrder = nullOrder;
- this.expressionDef = expressionDef;
}
public void reset(BoundaryDef bndDef) {
this.bndDef = bndDef;
}
+ protected abstract int computeStart(int rowIdx, PTFPartition p) throws HiveException;
+
+ protected abstract int computeEnd(int rowIdx, PTFPartition p) throws HiveException;
+ }
+
+ /*
+ * - starting from the given rowIdx scan in the given direction until a row's expr
+ * evaluates to an amt that crosses the 'amt' threshold specified in the ValueBoundaryDef.
+ */
+ static abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner {
+ OrderExpressionDef expressionDef;
+
+ public SingleValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef);
+ this.expressionDef = expressionDef;
+ }
+
/*
| Use | Boundary1.type | Boundary1. amt | Sort Key | Order | Behavior |
| Case | | | | | |
@@ -826,6 +834,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
| | | | | | such that R2.sk - R.sk > amt |
|------+----------------+----------------+----------+-------+-----------------------------------|
*/
+ @Override
protected int computeStart(int rowIdx, PTFPartition p) throws HiveException {
switch(bndDef.getDirection()) {
case PRECEDING:
@@ -838,9 +847,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- /*
- *
- */
protected int computeStartPreceding(int rowIdx, PTFPartition p) throws HiveException {
int amt = bndDef.getAmt();
// Use Case 1.
@@ -851,7 +857,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
if ( sortKey == null ) {
// Use Case 2.
- if ( order == Order.ASC ) {
+ if ( expressionDef.getOrder() == Order.ASC ) {
return 0;
}
else { // Use Case 3.
@@ -869,7 +875,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
int r = rowIdx;
// Use Case 4.
- if ( order == Order.DESC ) {
+ if ( expressionDef.getOrder() == Order.DESC ) {
while (r >= 0 && !isDistanceGreater(rowVal, sortKey, amt) ) {
r--;
if ( r >= 0 ) {
@@ -925,7 +931,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
if ( sortKey == null ) {
// Use Case 9.
- if ( order == Order.DESC) {
+ if ( expressionDef.getOrder() == Order.DESC) {
return p.size();
}
else { // Use Case 10.
@@ -940,7 +946,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
// Use Case 11.
- if ( order == Order.DESC) {
+ if ( expressionDef.getOrder() == Order.DESC) {
while (r < p.size() && !isDistanceGreater(sortKey, rowVal, amt) ) {
r++;
if ( r < p.size() ) {
@@ -992,6 +998,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
| | | | | | end = R2.idx |
|------+----------------+---------------+----------+-------+-----------------------------------|
*/
+ @Override
protected int computeEnd(int rowIdx, PTFPartition p) throws HiveException {
switch(bndDef.getDirection()) {
case PRECEDING:
@@ -1013,7 +1020,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
if ( sortKey == null ) {
// Use Case 2.
- if ( order == Order.DESC ) {
+ if ( expressionDef.getOrder() == Order.DESC ) {
return p.size();
}
else { // Use Case 3.
@@ -1025,7 +1032,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
int r = rowIdx;
// Use Case 4.
- if ( order == Order.DESC ) {
+ if ( expressionDef.getOrder() == Order.DESC ) {
while (r >= 0 && !isDistanceGreater(rowVal, sortKey, amt) ) {
r--;
if ( r >= 0 ) {
@@ -1086,7 +1093,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
if ( sortKey == null ) {
// Use Case 9.
- if ( order == Order.DESC) {
+ if ( expressionDef.getOrder() == Order.DESC) {
return p.size();
}
else { // Use Case 10.
@@ -1101,7 +1108,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
// Use Case 11.
- if ( order == Order.DESC) {
+ if ( expressionDef.getOrder() == Order.DESC) {
while (r < p.size() && !isDistanceGreater(sortKey, rowVal, amt) ) {
r++;
if ( r < p.size() ) {
@@ -1140,25 +1147,30 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
@SuppressWarnings("incomplete-switch")
- public static ValueBoundaryScanner getScanner(ValueBoundaryDef vbDef, Order order, NullOrder nullOrder)
+ public static SingleValueBoundaryScanner getScanner(ValueBoundaryDef vbDef)
throws HiveException {
- PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) vbDef.getOI();
+ if (vbDef.getOrderDef().getExpressions().size() != 1) {
+ throw new HiveException("Internal error: initializing SingleValueBoundaryScanner with"
+ + " multiple expression for sorting");
+ }
+ OrderExpressionDef exprDef = vbDef.getOrderDef().getExpressions().get(0);
+ PrimitiveObjectInspector pOI = (PrimitiveObjectInspector) exprDef.getOI();
switch(pOI.getPrimitiveCategory()) {
case BYTE:
case INT:
case LONG:
case SHORT:
case TIMESTAMP:
- return new LongValueBoundaryScanner(vbDef, order, nullOrder, vbDef.getExpressionDef());
+ return new LongValueBoundaryScanner(vbDef, exprDef);
case DOUBLE:
case FLOAT:
- return new DoubleValueBoundaryScanner(vbDef, order, nullOrder, vbDef.getExpressionDef());
+ return new DoubleValueBoundaryScanner(vbDef, exprDef);
case DECIMAL:
- return new HiveDecimalValueBoundaryScanner(vbDef, order, nullOrder, vbDef.getExpressionDef());
+ return new HiveDecimalValueBoundaryScanner(vbDef, exprDef);
case DATE:
- return new DateValueBoundaryScanner(vbDef, order, nullOrder, vbDef.getExpressionDef());
+ return new DateValueBoundaryScanner(vbDef, exprDef);
case STRING:
- return new StringValueBoundaryScanner(vbDef, order, nullOrder, vbDef.getExpressionDef());
+ return new StringValueBoundaryScanner(vbDef, exprDef);
}
throw new HiveException(
String.format("Internal Error: attempt to setup a Window for datatype %s",
@@ -1166,10 +1178,9 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- public static class LongValueBoundaryScanner extends ValueBoundaryScanner {
- public LongValueBoundaryScanner(BoundaryDef bndDef, Order order, NullOrder nullOrder,
- PTFExpressionDef expressionDef) {
- super(bndDef,order,nullOrder,expressionDef);
+ public static class LongValueBoundaryScanner extends SingleValueBoundaryScanner {
+ public LongValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef,expressionDef);
}
@Override
@@ -1199,10 +1210,9 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- public static class DoubleValueBoundaryScanner extends ValueBoundaryScanner {
- public DoubleValueBoundaryScanner(BoundaryDef bndDef, Order order,
- NullOrder nullOrder, PTFExpressionDef expressionDef) {
- super(bndDef,order,nullOrder,expressionDef);
+ public static class DoubleValueBoundaryScanner extends SingleValueBoundaryScanner {
+ public DoubleValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef,expressionDef);
}
@Override
@@ -1232,10 +1242,9 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- public static class HiveDecimalValueBoundaryScanner extends ValueBoundaryScanner {
- public HiveDecimalValueBoundaryScanner(BoundaryDef bndDef, Order order,
- NullOrder nullOrder, PTFExpressionDef expressionDef) {
- super(bndDef,order,nullOrder,expressionDef);
+ public static class HiveDecimalValueBoundaryScanner extends SingleValueBoundaryScanner {
+ public HiveDecimalValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef,expressionDef);
}
@Override
@@ -1265,10 +1274,9 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- public static class DateValueBoundaryScanner extends ValueBoundaryScanner {
- public DateValueBoundaryScanner(BoundaryDef bndDef, Order order,
- NullOrder nullOrder, PTFExpressionDef expressionDef) {
- super(bndDef,order,nullOrder,expressionDef);
+ public static class DateValueBoundaryScanner extends SingleValueBoundaryScanner {
+ public DateValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef,expressionDef);
}
@Override
@@ -1293,10 +1301,9 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
- public static class StringValueBoundaryScanner extends ValueBoundaryScanner {
- public StringValueBoundaryScanner(BoundaryDef bndDef, Order order,
- NullOrder nullOrder, PTFExpressionDef expressionDef) {
- super(bndDef,order,nullOrder,expressionDef);
+ public static class StringValueBoundaryScanner extends SingleValueBoundaryScanner {
+ public StringValueBoundaryScanner(BoundaryDef bndDef, OrderExpressionDef expressionDef) {
+ super(bndDef,expressionDef);
}
@Override
@@ -1318,6 +1325,149 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
+ /*
+ */
+ static class MultiValueBoundaryScanner extends ValueBoundaryScanner {
+ OrderDef orderDef;
+
+ public MultiValueBoundaryScanner(BoundaryDef bndDef, OrderDef orderDef) {
+ super(bndDef);
+ this.orderDef = orderDef;
+ }
+
+ /*
+|------+----------------+----------------+----------+-------+-----------------------------------|
+| Use | Boundary1.type | Boundary1. amt | Sort Key | Order | Behavior |
+| Case | | | | | |
+|------+----------------+----------------+----------+-------+-----------------------------------|
+| 1. | PRECEDING | UNB | ANY | ANY | start = 0 |
+| 2. | CURRENT ROW | | ANY | ANY | scan backwards until row R2 |
+| | | | | | such R2.sk != R.sk |
+| | | | | | start = R2.idx + 1 |
+|------+----------------+----------------+----------+-------+-----------------------------------|
+ */
+ @Override
+ protected int computeStart(int rowIdx, PTFPartition p) throws HiveException {
+ switch(bndDef.getDirection()) {
+ case PRECEDING:
+ return computeStartPreceding(rowIdx, p);
+ case CURRENT:
+ return computeStartCurrentRow(rowIdx, p);
+ case FOLLOWING:
+ default:
+ throw new HiveException(
+ "FOLLOWING not allowed for starting RANGE with multiple expressions in ORDER BY");
+ }
+ }
+
+ protected int computeStartPreceding(int rowIdx, PTFPartition p) throws HiveException {
+ int amt = bndDef.getAmt();
+ if ( amt == BoundarySpec.UNBOUNDED_AMOUNT ) {
+ return 0;
+ }
+ throw new HiveException(
+ "PRECEDING needs UNBOUNDED for RANGE with multiple expressions in ORDER BY");
+ }
+
+ protected int computeStartCurrentRow(int rowIdx, PTFPartition p) throws HiveException {
+ Object[] sortKey = computeValues(p.getAt(rowIdx));
+ Object[] rowVal = sortKey;
+ int r = rowIdx;
+
+ while (r >= 0 && isEqual(rowVal, sortKey) ) {
+ r--;
+ if ( r >= 0 ) {
+ rowVal = computeValues(p.getAt(r));
+ }
+ }
+ return r + 1;
+ }
+
+ /*
+|------+----------------+---------------+----------+-------+-----------------------------------|
+| Use | Boundary2.type | Boundary2.amt | Sort Key | Order | Behavior |
+| Case | | | | | |
+|------+----------------+---------------+----------+-------+-----------------------------------|
+| 1. | CURRENT ROW | | ANY | ANY | scan forward until row R2 |
+| | | | | | such that R2.sk != R.sk |
+| | | | | | end = R2.idx |
+| 2. | FOLLOWING | UNB | ANY | ANY | end = partition.size() |
+|------+----------------+---------------+----------+-------+-----------------------------------|
+ */
+ @Override
+ protected int computeEnd(int rowIdx, PTFPartition p) throws HiveException {
+ switch(bndDef.getDirection()) {
+ case PRECEDING:
+ throw new HiveException(
+ "PRECEDING not allowed for finishing RANGE with multiple expressions in ORDER BY");
+ case CURRENT:
+ return computeEndCurrentRow(rowIdx, p);
+ case FOLLOWING:
+ default:
+ return computeEndFollowing(rowIdx, p);
+ }
+ }
+
+ protected int computeEndCurrentRow(int rowIdx, PTFPartition p) throws HiveException {
+ Object[] sortKey = computeValues(p.getAt(rowIdx));
+ Object[] rowVal = sortKey;
+ int r = rowIdx;
+
+ while (r < p.size() && isEqual(sortKey, rowVal) ) {
+ r++;
+ if ( r < p.size() ) {
+ rowVal = computeValues(p.getAt(r));
+ }
+ }
+ return r;
+ }
+
+ protected int computeEndFollowing(int rowIdx, PTFPartition p) throws HiveException {
+ int amt = bndDef.getAmt();
+ if ( amt == BoundarySpec.UNBOUNDED_AMOUNT ) {
+ return p.size();
+ }
+ throw new HiveException(
+ "FOLLOWING needs UNBOUNDED for RANGE with multiple expressions in ORDER BY");
+ }
+
+ public Object[] computeValues(Object row) throws HiveException {
+ Object[] objs = new Object[orderDef.getExpressions().size()];
+ for (int i = 0; i < objs.length; i++) {
+ Object o = orderDef.getExpressions().get(i).getExprEvaluator().evaluate(row);
+ objs[i] = ObjectInspectorUtils.copyToStandardObject(o, orderDef.getExpressions().get(i).getOI());
+ }
+ return objs;
+ }
+
+ public boolean isEqual(Object[] v1, Object[] v2) {
+ assert v1.length == v2.length;
+ for (int i = 0; i < v1.length; i++) {
+ if (v1[i] == null && v2[i] == null) {
+ continue;
+ }
+ if (v1[i] == null || v2[i] == null) {
+ return false;
+ }
+ if (ObjectInspectorUtils.compare(
+ v1[i], orderDef.getExpressions().get(i).getOI(),
+ v2[i], orderDef.getExpressions().get(i).getOI()) != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static MultiValueBoundaryScanner getScanner(ValueBoundaryDef vbDef)
+ throws HiveException {
+ if (vbDef.getOrderDef().getExpressions().size() <= 1) {
+ throw new HiveException("Internal error: initializing SingleValueBoundaryScanner with"
+ + " multiple expression for sorting");
+ }
+ return new MultiValueBoundaryScanner(vbDef, vbDef.getOrderDef());
+ }
+ }
+
public static class SameList<E> extends AbstractList<E> {
int sz;
E val;
@@ -1351,8 +1501,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
*/
int[] wFnsToProcess;
WindowTableFunctionDef wTFnDef;
- Order order;
- NullOrder nullOrder;
PTFDesc ptfDesc;
StructObjectInspector inputOI;
AggregationBuffer[] aggBuffers;
@@ -1367,8 +1515,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
this.wFnsToProcess = wFnsToProcess;
this.currIdx = 0;
wTFnDef = (WindowTableFunctionDef) getTableDef();
- order = wTFnDef.getOrder().getExpressions().get(0).getOrder();
- nullOrder = wTFnDef.getOrder().getExpressions().get(0).getNullOrder();
ptfDesc = getQueryDef();
inputOI = iPart.getOutputOI();
@@ -1423,7 +1569,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
out = ObjectInspectorUtils.copyToStandardObject(out, wFn.getOI());
output.set(j, out);
} else {
- Range rng = getRange(wFn, currIdx, iPart, order, nullOrder);
+ Range rng = getRange(wFn, currIdx, iPart);
PTFPartitionIterator<Object> rItr = rng.iterator();
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, rItr);
output.set(j, evaluateWindowFunction(wFn, rItr));
@@ -1459,8 +1605,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
List<Object>[] fnOutputs;
AggregationBuffer[] aggBuffers;
Object[][] funcArgs;
- Order order;
- NullOrder nullOrder;
RankLimit rnkLimit;
@SuppressWarnings("unchecked")
@@ -1474,9 +1618,6 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI,
precedingSpan, followingSpan);
- order = tabDef.getOrder().getExpressions().get(0).getOrder();
- nullOrder = tabDef.getOrder().getExpressions().get(0).getNullOrder();
-
int numFns = tabDef.getWindowFunctions().size();
fnOutputs = new ArrayList[numFns];
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_range_multiorder.q b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
new file mode 100644
index 0000000..d8ca4d6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
@@ -0,0 +1,34 @@
+drop table over10k;
+
+create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|';
+
+load data local inpath '../../data/files/over10k' into table over10k;
+
+select first_value(t) over ( partition by si order by i, b ) from over10k limit 100;
+
+select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k limit 100;
+
+select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k limit 100;
+
+select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k limit 100;
+
+select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100;
+
+select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100;
+
+select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k limit 100;
+
+select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/0b574501/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
new file mode 100644
index 0000000..9910883
--- /dev/null
+++ b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
@@ -0,0 +1,910 @@
+PREHOOK: query: drop table over10k
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table over10k
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over10k
+POSTHOOK: query: create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over10k
+PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@over10k
+POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@over10k
+PREHOOK: query: select first_value(t) over ( partition by si order by i, b ) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select first_value(t) over ( partition by si order by i, b ) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+51
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+48
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+47
+PREHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+65543
+65549
+65558
+65580
+65586
+65596
+65616
+65620
+65627
+65640
+65643
+65706
+65713
+65737
+65744
+65752
+65778
+65540
+65563
+65599
+65604
+65613
+65613
+65615
+65651
+65653
+65668
+65693
+65731
+65733
+65738
+65741
+65744
+65747
+65763
+65778
+65789
+65541
+65547
+65560
+65572
+65574
+65575
+65578
+65588
+65594
+65610
+65691
+65694
+65711
+65719
+65722
+65738
+65756
+65790
+65542
+65557
+65566
+65584
+65610
+65612
+65626
+65631
+65638
+65654
+65654
+65655
+65699
+65712
+65720
+65732
+65748
+65752
+65771
+65771
+65771
+65781
+65565
+65569
+65573
+65582
+65584
+65606
+65656
+65669
+65717
+65724
+65728
+65761
+65762
+65770
+65771
+65781
+65546
+65551
+65551
+65568
+65568
+65579
+65603
+PREHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+1
+2
+3
+4
+5
+6
+7
+PREHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+alice allen 451 65662 65640.125
+alice allen 462 65545 65640.125
+alice allen 501 65720 65640.125
+alice allen 501 65670 65640.125
+alice allen 484 65600 65640.125
+alice allen 472 65609 65640.125
+alice allen 509 65758 65640.125
+alice allen 400 65557 65640.125
+alice brown 425 65570 65696.71428571429
+alice brown 376 65708 65696.71428571429
+alice brown 324 65569 65696.71428571429
+alice brown 302 65711 65696.71428571429
+alice brown 381 65704 65696.71428571429
+alice brown 452 65666 65696.71428571429
+alice brown 346 65696 65696.71428571429
+alice brown 471 65733 65696.71428571429
+alice brown 409 65667 65696.71428571429
+alice brown 399 65779 65696.71428571429
+alice brown 332 65781 65696.71428571429
+alice brown 337 65707 65696.71428571429
+alice brown 499 65790 65696.71428571429
+alice brown 492 65673 65696.71428571429
+alice carson 404 65710 65645.4
+alice carson 376 65576 65645.4
+alice carson 508 65545 65645.4
+alice carson 427 65559 65645.4
+alice carson 473 65565 65645.4
+alice carson 390 65747 65645.4
+alice carson 318 65695 65645.4
+alice carson 316 65559 65645.4
+alice carson 268 65713 65645.4
+alice carson 380 65785 65645.4
+alice davidson 298 65554 65648.5
+alice davidson 479 65631 65648.5
+alice davidson 445 65590 65648.5
+alice davidson 384 65676 65648.5
+alice davidson 408 65791 65648.5
+alice davidson 321 65677 65648.5
+alice davidson 448 65641 65648.5
+alice davidson 423 65740 65648.5
+alice davidson 270 65563 65648.5
+alice davidson 431 65677 65648.5
+alice davidson 487 65596 65648.5
+alice davidson 402 65544 65648.5
+alice davidson 272 65742 65648.5
+alice davidson 287 65747 65648.5
+alice davidson 328 65547 65648.5
+alice davidson 437 65690 65648.5
+alice davidson 308 65560 65648.5
+alice davidson 408 65707 65648.5
+alice ellison 405 65713 65669.13333333333
+alice ellison 490 65572 65669.13333333333
+alice ellison 354 65698 65669.13333333333
+alice ellison 331 65557 65669.13333333333
+alice ellison 313 65612 65669.13333333333
+alice ellison 296 65741 65669.13333333333
+alice ellison 403 65544 65669.13333333333
+alice ellison 482 65681 65669.13333333333
+alice ellison 320 65745 65669.13333333333
+alice ellison 274 65537 65669.13333333333
+alice ellison 256 65744 65669.13333333333
+alice ellison 355 65699 65669.13333333333
+alice ellison 343 65787 65669.13333333333
+alice ellison 335 65730 65669.13333333333
+alice ellison 374 65677 65669.13333333333
+alice falkner 342 65752 65695.76470588235
+alice falkner 280 65597 65695.76470588235
+alice falkner 393 65611 65695.76470588235
+alice falkner 389 65699 65695.76470588235
+alice falkner 345 65773 65695.76470588235
+alice falkner 500 65775 65695.76470588235
+alice falkner 323 65669 65695.76470588235
+alice falkner 393 65685 65695.76470588235
+alice falkner 339 65785 65695.76470588235
+alice falkner 382 65690 65695.76470588235
+alice falkner 371 65710 65695.76470588235
+alice falkner 481 65709 65695.76470588235
+alice falkner 311 65715 65695.76470588235
+alice falkner 477 65722 65695.76470588235
+alice falkner 382 65622 65695.76470588235
+alice falkner 455 65718 65695.76470588235
+alice falkner 452 65596 65695.76470588235
+alice garcia 388 65675 65688.76923076923
+alice garcia 366 65744 65688.76923076923
+alice garcia 331 65734 65688.76923076923
+alice garcia 299 65623 65688.76923076923
+alice garcia 379 65746 65688.76923076923
+alice garcia 486 65725 65688.76923076923
+alice garcia 427 65674 65688.76923076923
+alice garcia 263 65630 65688.76923076923
+alice garcia 459 65712 65688.76923076923
+alice garcia 446 65759 65688.76923076923
+alice garcia 325 65573 65688.76923076923
+alice garcia 309 65746 65688.76923076923
+alice garcia 446 65613 65688.76923076923
+alice hernandez 396 65545 65678.38888888889
+alice hernandez 336 65786 65678.38888888889
+alice hernandez 324 65720 65678.38888888889
+alice hernandez 270 65717 65678.38888888889
+alice hernandez 323 65727 65678.38888888889
+PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+alice allen 400 65557 65557.0
+alice allen 451 65662 65609.5
+alice allen 462 65545 65588.0
+alice allen 472 65609 65593.25
+alice allen 484 65600 65594.6
+alice allen 501 65670 65607.16666666667
+alice allen 501 65720 65623.28571428571
+alice allen 509 65758 65640.125
+alice brown 302 65711 65711.0
+alice brown 324 65569 65640.0
+alice brown 332 65781 65687.0
+alice brown 337 65707 65692.0
+alice brown 346 65696 65692.8
+alice brown 376 65708 65695.33333333333
+alice brown 381 65704 65696.57142857143
+alice brown 399 65779 65706.875
+alice brown 409 65667 65702.44444444444
+alice brown 425 65570 65689.2
+alice brown 452 65666 65687.09090909091
+alice brown 471 65733 65690.91666666667
+alice brown 492 65673 65689.53846153847
+alice brown 499 65790 65696.71428571429
+alice carson 268 65713 65713.0
+alice carson 316 65559 65636.0
+alice carson 318 65695 65655.66666666667
+alice carson 376 65576 65635.75
+alice carson 380 65785 65665.6
+alice carson 390 65747 65679.16666666667
+alice carson 404 65710 65683.57142857143
+alice carson 427 65559 65668.0
+alice carson 473 65565 65656.55555555556
+alice carson 508 65545 65645.4
+alice davidson 270 65563 65563.0
+alice davidson 272 65742 65652.5
+alice davidson 287 65747 65684.0
+alice davidson 298 65554 65651.5
+alice davidson 308 65560 65633.2
+alice davidson 321 65677 65640.5
+alice davidson 328 65547 65627.14285714286
+alice davidson 384 65676 65633.25
+alice davidson 402 65544 65623.33333333333
+alice davidson 408 65707 65631.7
+alice davidson 408 65791 65646.18181818182
+alice davidson 423 65740 65654.0
+alice davidson 431 65677 65655.76923076923
+alice davidson 437 65690 65658.21428571429
+alice davidson 445 65590 65653.66666666667
+alice davidson 448 65641 65652.875
+alice davidson 479 65631 65651.58823529411
+alice davidson 487 65596 65648.5
+alice ellison 256 65744 65744.0
+alice ellison 274 65537 65640.5
+alice ellison 296 65741 65674.0
+alice ellison 313 65612 65658.5
+alice ellison 320 65745 65675.8
+alice ellison 331 65557 65656.0
+alice ellison 335 65730 65666.57142857143
+alice ellison 343 65787 65681.625
+alice ellison 354 65698 65683.44444444444
+alice ellison 355 65699 65685.0
+alice ellison 374 65677 65684.27272727272
+alice ellison 403 65544 65672.58333333333
+alice ellison 405 65713 65675.69230769231
+alice ellison 482 65681 65676.07142857143
+alice ellison 490 65572 65669.13333333333
+alice falkner 280 65597 65597.0
+alice falkner 311 65715 65656.0
+alice falkner 323 65669 65660.33333333333
+alice falkner 339 65785 65691.5
+alice falkner 342 65752 65703.6
+alice falkner 345 65773 65715.16666666667
+alice falkner 371 65710 65714.42857142857
+alice falkner 382 65622 65702.875
+alice falkner 382 65690 65701.44444444444
+alice falkner 389 65699 65701.2
+alice falkner 393 65611 65693.0
+alice falkner 393 65685 65692.33333333333
+alice falkner 452 65596 65684.92307692308
+alice falkner 455 65718 65687.28571428571
+alice falkner 477 65722 65689.6
+alice falkner 481 65709 65690.8125
+alice falkner 500 65775 65695.76470588235
+alice garcia 263 65630 65630.0
+alice garcia 299 65623 65626.5
+alice garcia 309 65746 65666.33333333333
+alice garcia 325 65573 65643.0
+alice garcia 331 65734 65661.2
+alice garcia 366 65744 65675.0
+alice garcia 379 65746 65685.14285714286
+alice garcia 388 65675 65683.875
+alice garcia 427 65674 65682.77777777778
+alice garcia 446 65613 65675.8
+alice garcia 446 65759 65683.36363636363
+alice garcia 459 65712 65685.75
+alice garcia 486 65725 65688.76923076923
+alice hernandez 270 65717 65717.0
+alice hernandez 290 65685 65701.0
+alice hernandez 296 65569 65657.0
+alice hernandez 320 65700 65667.75
+alice hernandez 323 65727 65679.6
+PREHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+alice allen 400 65557 65557
+alice allen 451 65662 65557
+alice allen 462 65545 65545
+alice allen 472 65609 65545
+alice allen 484 65600 65545
+alice allen 501 65670 65545
+alice allen 501 65720 65545
+alice allen 509 65758 65545
+alice brown 302 65711 65711
+alice brown 324 65569 65569
+alice brown 332 65781 65569
+alice brown 337 65707 65569
+alice brown 346 65696 65569
+alice brown 376 65708 65569
+alice brown 381 65704 65569
+alice brown 399 65779 65569
+alice brown 409 65667 65569
+alice brown 425 65570 65569
+alice brown 452 65666 65569
+alice brown 471 65733 65569
+alice brown 492 65673 65569
+alice brown 499 65790 65569
+alice carson 268 65713 65713
+alice carson 316 65559 65559
+alice carson 318 65695 65559
+alice carson 376 65576 65559
+alice carson 380 65785 65559
+alice carson 390 65747 65559
+alice carson 404 65710 65559
+alice carson 427 65559 65559
+alice carson 473 65565 65559
+alice carson 508 65545 65545
+alice davidson 270 65563 65563
+alice davidson 272 65742 65563
+alice davidson 287 65747 65563
+alice davidson 298 65554 65554
+alice davidson 308 65560 65554
+alice davidson 321 65677 65554
+alice davidson 328 65547 65547
+alice davidson 384 65676 65547
+alice davidson 402 65544 65544
+alice davidson 408 65707 65544
+alice davidson 408 65791 65544
+alice davidson 423 65740 65544
+alice davidson 431 65677 65544
+alice davidson 437 65690 65544
+alice davidson 445 65590 65544
+alice davidson 448 65641 65544
+alice davidson 479 65631 65544
+alice davidson 487 65596 65544
+alice ellison 256 65744 65744
+alice ellison 274 65537 65537
+alice ellison 296 65741 65537
+alice ellison 313 65612 65537
+alice ellison 320 65745 65537
+alice ellison 331 65557 65537
+alice ellison 335 65730 65537
+alice ellison 343 65787 65537
+alice ellison 354 65698 65537
+alice ellison 355 65699 65537
+alice ellison 374 65677 65537
+alice ellison 403 65544 65537
+alice ellison 405 65713 65537
+alice ellison 482 65681 65537
+alice ellison 490 65572 65537
+alice falkner 280 65597 65597
+alice falkner 311 65715 65597
+alice falkner 323 65669 65597
+alice falkner 339 65785 65597
+alice falkner 342 65752 65597
+alice falkner 345 65773 65597
+alice falkner 371 65710 65597
+alice falkner 382 65622 65597
+alice falkner 382 65690 65597
+alice falkner 389 65699 65597
+alice falkner 393 65611 65597
+alice falkner 393 65685 65597
+alice falkner 452 65596 65596
+alice falkner 455 65718 65596
+alice falkner 477 65722 65596
+alice falkner 481 65709 65596
+alice falkner 500 65775 65596
+alice garcia 263 65630 65630
+alice garcia 299 65623 65623
+alice garcia 309 65746 65623
+alice garcia 325 65573 65573
+alice garcia 331 65734 65573
+alice garcia 366 65744 65573
+alice garcia 379 65746 65573
+alice garcia 388 65675 65573
+alice garcia 427 65674 65573
+alice garcia 446 65613 65573
+alice garcia 446 65759 65573
+alice garcia 459 65712 65573
+alice garcia 486 65725 65573
+alice hernandez 270 65717 65717
+alice hernandez 290 65685 65685
+alice hernandez 296 65569 65569
+alice hernandez 320 65700 65569
+alice hernandez 323 65727 65569
+PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+alice allen 400 65557 65557.0
+alice allen 451 65662 65609.5
+alice allen 462 65545 65588.0
+alice allen 472 65609 65593.25
+alice allen 484 65600 65594.6
+alice allen 501 65720 65615.5
+alice allen 501 65670 65623.28571428571
+alice allen 509 65758 65640.125
+alice brown 302 65711 65711.0
+alice brown 324 65569 65640.0
+alice brown 332 65781 65687.0
+alice brown 337 65707 65692.0
+alice brown 346 65696 65692.8
+alice brown 376 65708 65695.33333333333
+alice brown 381 65704 65696.57142857143
+alice brown 399 65779 65706.875
+alice brown 409 65667 65702.44444444444
+alice brown 425 65570 65689.2
+alice brown 452 65666 65687.09090909091
+alice brown 471 65733 65690.91666666667
+alice brown 492 65673 65689.53846153847
+alice brown 499 65790 65696.71428571429
+alice carson 268 65713 65713.0
+alice carson 316 65559 65636.0
+alice carson 318 65695 65655.66666666667
+alice carson 376 65576 65635.75
+alice carson 380 65785 65665.6
+alice carson 390 65747 65679.16666666667
+alice carson 404 65710 65683.57142857143
+alice carson 427 65559 65668.0
+alice carson 473 65565 65656.55555555556
+alice carson 508 65545 65645.4
+alice davidson 270 65563 65563.0
+alice davidson 272 65742 65652.5
+alice davidson 287 65747 65684.0
+alice davidson 298 65554 65651.5
+alice davidson 308 65560 65633.2
+alice davidson 321 65677 65640.5
+alice davidson 328 65547 65627.14285714286
+alice davidson 384 65676 65633.25
+alice davidson 402 65544 65623.33333333333
+alice davidson 408 65791 65640.1
+alice davidson 408 65707 65646.18181818182
+alice davidson 423 65740 65654.0
+alice davidson 431 65677 65655.76923076923
+alice davidson 437 65690 65658.21428571429
+alice davidson 445 65590 65653.66666666667
+alice davidson 448 65641 65652.875
+alice davidson 479 65631 65651.58823529411
+alice davidson 487 65596 65648.5
+alice ellison 256 65744 65744.0
+alice ellison 274 65537 65640.5
+alice ellison 296 65741 65674.0
+alice ellison 313 65612 65658.5
+alice ellison 320 65745 65675.8
+alice ellison 331 65557 65656.0
+alice ellison 335 65730 65666.57142857143
+alice ellison 343 65787 65681.625
+alice ellison 354 65698 65683.44444444444
+alice ellison 355 65699 65685.0
+alice ellison 374 65677 65684.27272727272
+alice ellison 403 65544 65672.58333333333
+alice ellison 405 65713 65675.69230769231
+alice ellison 482 65681 65676.07142857143
+alice ellison 490 65572 65669.13333333333
+alice falkner 280 65597 65597.0
+alice falkner 311 65715 65656.0
+alice falkner 323 65669 65660.33333333333
+alice falkner 339 65785 65691.5
+alice falkner 342 65752 65703.6
+alice falkner 345 65773 65715.16666666667
+alice falkner 371 65710 65714.42857142857
+alice falkner 382 65690 65711.375
+alice falkner 382 65622 65701.44444444444
+alice falkner 389 65699 65701.2
+alice falkner 393 65685 65699.72727272728
+alice falkner 393 65611 65692.33333333333
+alice falkner 452 65596 65684.92307692308
+alice falkner 455 65718 65687.28571428571
+alice falkner 477 65722 65689.6
+alice falkner 481 65709 65690.8125
+alice falkner 500 65775 65695.76470588235
+alice garcia 263 65630 65630.0
+alice garcia 299 65623 65626.5
+alice garcia 309 65746 65666.33333333333
+alice garcia 325 65573 65643.0
+alice garcia 331 65734 65661.2
+alice garcia 366 65744 65675.0
+alice garcia 379 65746 65685.14285714286
+alice garcia 388 65675 65683.875
+alice garcia 427 65674 65682.77777777778
+alice garcia 446 65759 65690.4
+alice garcia 446 65613 65683.36363636363
+alice garcia 459 65712 65685.75
+alice garcia 486 65725 65688.76923076923
+alice hernandez 270 65717 65717.0
+alice hernandez 290 65685 65701.0
+alice hernandez 296 65569 65657.0
+alice hernandez 320 65700 65667.75
+alice hernandez 323 65727 65679.6
+PREHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+256 false 65543 32.21 65543
+256 false 65549 23.72 65549
+256 false 65558 71.32 65558
+256 false 65580 64.81 65580
+256 false 65586 12.97 65586
+256 false 65596 5.35 65596
+256 false 65616 76.38 65616
+256 false 65620 51.72 65620
+256 false 65627 54.23 65627
+256 false 65640 32.64 65640
+256 false 65643 94.05 65643
+256 false 65706 83.67 65706
+256 false 65713 21.83 65713
+256 false 65737 3.38 65737
+256 false 65744 47.17 65744
+256 false 65752 61.21 65752
+256 false 65778 16.29 65778
+256 true 65540 49.44 65540
+256 true 65563 94.87 65563
+256 true 65599 89.55 65599
+256 true 65604 40.97 65604
+256 true 65613 93.29 65613
+256 true 65613 78.27 65613
+256 true 65615 20.66 65615
+256 true 65651 90.32 65651
+256 true 65653 8.1 65653
+256 true 65668 92.71 65668
+256 true 65693 62.52 65693
+256 true 65731 34.09 65731
+256 true 65733 70.53 65733
+256 true 65738 9.0 65738
+256 true 65741 54.8 65741
+256 true 65744 38.16 65744
+256 true 65747 32.18 65747
+256 true 65763 24.89 65763
+256 true 65778 74.15 65778
+256 true 65789 91.12 65789
+257 false 65541 51.26 65541
+257 false 65547 54.01 65547
+257 false 65560 42.14 65560
+257 false 65572 79.15 65572
+257 false 65574 19.96 65574
+257 false 65575 1.21 65575
+257 false 65578 61.6 65578
+257 false 65588 81.17 65588
+257 false 65594 78.39 65594
+257 false 65610 98.0 65610
+257 false 65691 80.76 65691
+257 false 65694 29.0 65694
+257 false 65711 60.88 65711
+257 false 65719 62.79 65719
+257 false 65722 79.05 65722
+257 false 65738 96.01 65738
+257 false 65756 24.44 65756
+257 false 65790 9.26 65790
+257 true 65542 62.59 65542
+257 true 65557 55.07 65557
+257 true 65566 68.54 65566
+257 true 65584 35.88 65584
+257 true 65610 47.58 65610
+257 true 65612 3.12 65612
+257 true 65626 23.18 65626
+257 true 65631 51.61 65631
+257 true 65638 95.35 65638
+257 true 65654 24.54 65654
+257 true 65654 9.8 65654
+257 true 65655 40.42 65655
+257 true 65699 15.36 65699
+257 true 65712 90.44 65712
+257 true 65720 24.4 65720
+257 true 65732 96.85 65732
+257 true 65748 32.52 65748
+257 true 65752 49.35 65752
+257 true 65771 95.58 65771
+257 true 65771 53.89 65771
+257 true 65771 48.5 65771
+257 true 65781 17.33 65781
+258 false 65565 98.19 65565
+258 false 65569 66.81 65569
+258 false 65573 31.45 65573
+258 false 65582 67.28 65582
+258 false 65584 64.92 65584
+258 false 65606 35.52 65606
+258 false 65656 79.17 65656
+258 false 65669 75.01 65669
+258 false 65717 95.76 65717
+258 false 65724 70.0 65724
+258 false 65728 9.05 65728
+258 false 65761 33.73 65761
+258 false 65762 15.22 65762
+258 false 65770 13.38 65770
+258 false 65771 52.63 65771
+258 false 65781 1.92 65781
+258 true 65546 91.19 65546
+258 true 65551 91.56 65551
+258 true 65551 88.97 65551
+258 true 65568 81.41 65568
+258 true 65568 13.57 65568
+258 true 65579 47.52 65579
+258 true 65603 2.61 65603