You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2026/07/13 12:50:16 UTC
(hive) branch master updated: HIVE-29696: Vectorization: ClassCastException in VectorPTFGroupBatches when buffering complex window function inputs (#6579)
This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d2d7ff14eef HIVE-29696: Vectorization: ClassCastException in VectorPTFGroupBatches when buffering complex window function inputs (#6579)
d2d7ff14eef is described below
commit d2d7ff14eefebacf296008bb5d03f620fcc3cd72
Author: Raghav Aggarwal <ra...@gmail.com>
AuthorDate: Mon Jul 13 18:20:10 2026 +0530
HIVE-29696: Vectorization: ClassCastException in VectorPTFGroupBatches when buffering complex window function inputs (#6579)
---
.../vector/ptf/BufferedVectorizedRowBatch.java | 1 -
.../ql/exec/vector/ptf/VectorPTFEvaluatorBase.java | 6 -
.../exec/vector/ptf/VectorPTFEvaluatorCount.java | 1 -
.../ptf/VectorPTFEvaluatorCountDistinct.java | 1 -
.../vector/ptf/VectorPTFEvaluatorCountStar.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalAvg.java | 1 -
.../ptf/VectorPTFEvaluatorDecimalFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorDecimalLastValue.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalMax.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalMin.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalSum.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleAvg.java | 1 -
.../ptf/VectorPTFEvaluatorDoubleFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorDoubleLastValue.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleMax.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleMin.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleSum.java | 1 -
.../ql/exec/vector/ptf/VectorPTFEvaluatorLag.java | 2 +-
.../ql/exec/vector/ptf/VectorPTFEvaluatorLead.java | 2 +-
.../exec/vector/ptf/VectorPTFEvaluatorLongAvg.java | 1 -
.../ptf/VectorPTFEvaluatorLongFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorLongLastValue.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongMax.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongMin.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongSum.java | 1 -
.../vector/ptf/VectorPTFEvaluatorRowNumber.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalSum.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleSum.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongSum.java | 1 -
.../ql/exec/vector/ptf/VectorPTFGroupBatches.java | 12 -
.../hive/ql/exec/vector/ptf/VectorPTFOperator.java | 61 ++-
.../exec/vector/ptf/VectorSpillBlockContainer.java | 5 -
.../clientpositive/vector_ptf_nested_input_expr.q | 56 +++
.../llap/vector_ptf_nested_input_expr.q.out | 434 +++++++++++++++++++++
43 files changed, 522 insertions(+), 91 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
index f70fc2dea47..2e750f6408d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
@@ -24,7 +24,6 @@
*/
public class BufferedVectorizedRowBatch extends VectorizedRowBatch {
boolean isLastGroupBatch;
- boolean isInputExpressionEvaluated;
public BufferedVectorizedRowBatch(int numCols) {
super(numCols);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
index 9243c3cb00e..31995c434d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
@@ -86,12 +86,6 @@ public void setNullsLast(boolean nullsLast) {
this.nullsLast = nullsLast;
}
- // Evaluate the aggregation input argument expression.
- public void evaluateInputExpr(VectorizedRowBatch batch) throws HiveException {
- if (inputVecExpr != null) {
- inputVecExpr.evaluate(batch);
- }
- }
/*
* Evaluate the aggregation over one of the group's batches. In this mode, evaluator is not
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
index 94b038e2e8e..eed564b3971 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorCount(WindowFrameDef windowFrameDef, VectorExpression i
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Count non-null column rows.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
index 167c07dc209..3f34274cf01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
@@ -68,7 +68,6 @@ public VectorPTFEvaluatorCountDistinct(WindowFrameDef windowFrameDef,
@Override
public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException {
- evaluateInputExpr(batch);
// We do not filter when PTF is in reducer.
Preconditions.checkState(!batch.selectedInUse);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
index c48141ef8d1..3cff308550c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorCountStar(WindowFrameDef windowFrameDef, VectorExpressi
@Override
public void evaluateGroupBatch(VectorizedRowBatch batch) {
// No input expression for COUNT(*).
- // evaluateInputExpr(batch);
// Count all rows.
count += batch.size;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
index 5d4d757060d..7f7cfec9d0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
@@ -54,7 +54,6 @@ public VectorPTFEvaluatorDecimalAvg(WindowFrameDef windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
index d21e882084b..c500cf193ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalFirstValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and decimal firstValue; stream fill result as repeated.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
index 0fb88d69c3d..78ff8b7bd20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalLastValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and decimal lastValue.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
index 11f146db706..33a8a8b4ddb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalMax(WindowFrameDef windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null decimal column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
index e06d1868856..bb57159338f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorDecimalMin(WindowFrameDef windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null decimal column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
index f9263e17af9..02b388d55dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalSum(WindowFrameDef windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
index d7123fb2579..7cd42a382c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorDoubleAvg(WindowFrameDef windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
index 40b49d7960c..026baa0aa33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
@@ -49,7 +49,6 @@ public VectorPTFEvaluatorDoubleFirstValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and double firstValue; stream fill result as repeated.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
index e7e4a0ce3a7..d8936413678 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
@@ -47,7 +47,6 @@ public VectorPTFEvaluatorDoubleLastValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and double lastValue.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
index 5b124d20269..1a8f029639d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMax(WindowFrameDef windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null double column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
index b76df013b35..e8f28eb9c31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMin(WindowFrameDef windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null double column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
index e8cda454bcd..4ecefb4f2fe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorDoubleSum(WindowFrameDef windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
index d35d089c6c0..31922508862 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
@@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range, VectorPTFGroupBatches batches)
if (lagRow < 0 || lagRow >= batches.size()) {
return getDefaultValue(rowNum, batches);
} else {
- return batches.getValueAndEvaluateInputExpression(this, lagRow, inputColumnNum);
+ return batches.getValue(lagRow, inputColumnNum);
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
index db0332ce9d9..d2806bb65b9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
@@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range, VectorPTFGroupBatches batches)
if (leadRow < 0 || leadRow >= batches.size()) {
return getDefaultValue(rowNum, batches);
} else {
- return batches.getValueAndEvaluateInputExpression(this, leadRow, inputColumnNum);
+ return batches.getValue(leadRow, inputColumnNum);
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
index 9c5b402fa5c..5729a9a11e2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorLongAvg(WindowFrameDef windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
index 1b1ca204939..d370a07880d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
@@ -49,7 +49,6 @@ public VectorPTFEvaluatorLongFirstValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and long firstValue; stream fill result as repeated.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
index bf0f27af832..536c05c9901 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorLongLastValue(WindowFrameDef windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and long lastValue.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
index 47911af4521..fe6d3753839 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMax(WindowFrameDef windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null long column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
index 91d21f2f182..2867653d10b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMin(WindowFrameDef windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null long column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
index d37bf985dc1..5318407d6bb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorLongSum(WindowFrameDef windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
index 3e4bc3d84bc..b287639c27a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
@@ -44,7 +44,6 @@ public VectorPTFEvaluatorRowNumber(WindowFrameDef windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
final int size = batch.size;
LongColumnVector longColVector = (LongColumnVector) batch.cols[outputColumnNum];
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
index e51d1fc5f66..58ad82003c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
@@ -56,7 +56,6 @@ public VectorPTFEvaluatorStreamingDecimalAvg(WindowFrameDef windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
index 93572427107..dabfa18b323 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMax(WindowFrameDef windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null decimal column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
index 51b43d7c622..a729775b00f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMin(WindowFrameDef windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null decimal column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
index bc8620ac37f..3fb86f9379d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalSum(WindowFrameDef windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values; maintain isGroupResultNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
index f6c5942c065..18d521c07c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingDoubleAvg(WindowFrameDef windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
index 1d61cc5bf09..5a40b638e71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMax(WindowFrameDef windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null double column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
index 9ac197d17a8..14c0b831edd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMin(WindowFrameDef windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null double column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
index 8f17663a358..2e0dfa9a45f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleSum(WindowFrameDef windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// We do not filter when PTF is in reducer.
Preconditions.checkState(!batch.selectedInUse);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
index 78d543a0c20..9117ac16d31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingLongAvg(WindowFrameDef windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values for avg; maintain isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
index 94d19b3376f..ee2804b5838 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMax(WindowFrameDef windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null long column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
index 2d7caf34152..42ab267e947 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMin(WindowFrameDef windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null long column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
index 76bca6b6557..311707966c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongSum(WindowFrameDef windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values; maintain isNull.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
index 8fee32a569c..a6547a11bcb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
@@ -346,15 +346,6 @@ public Object getValue(int row, int col) throws HiveException {
return getValueFromBatch(bufferedBatches.get(rp.batchIndex), col, rp.rowIndexInBatch);
}
- public Object getValueAndEvaluateInputExpression(VectorPTFEvaluatorBase evaluator, int row,
- int col) throws HiveException {
- RowPositionInBatch rp = getPosition(row);
- BufferedVectorizedRowBatch batch = bufferedBatches.get(rp.batchIndex);
- if (!batch.isInputExpressionEvaluated) {
- evaluator.evaluateInputExpr(batch);
- }
- return getValueFromBatch(batch, col, rp.rowIndexInBatch);
- }
public RowPositionInBatch getPosition(int i) throws HiveException {
if (positionCache[i] != null) {
@@ -1012,7 +1003,6 @@ private VectorizedRowBatch createTruncatedBufferedBatch(BufferedVectorizedRowBat
partialBatch.size = size;
partialBatch.isLastGroupBatch = bufferedBatch.isLastGroupBatch;
- partialBatch.isInputExpressionEvaluated = bufferedBatch.isInputExpressionEvaluated;
return partialBatch;
}
@@ -1174,7 +1164,6 @@ private void readBlockFromSpillToMemory(VectorSpillBlock block) throws HiveExcep
block.readSingleRowFromBytesContainer(bufferedBatch);
bufferedBatch.size += 1;
bufferedBatch.isLastGroupBatch = block.isLastGroupBatch[batchIndex];
- bufferedBatch.isInputExpressionEvaluated = block.isInputExpressionEvaluated[batchIndex];
}
currentBufferedBatchCount = block.spillBatchCount;
partitionMetrics.batchesReadFromSpill += currentBufferedBatchCount;
@@ -1202,7 +1191,6 @@ private void spillAndResetCurrentBufferedBatches() throws HiveException {
BufferedVectorizedRowBatch bufferedBatch = bufferedBatches.get(i);
block.spillBatch(bufferedBatch);
block.isLastGroupBatch[i] = bufferedBatch.isLastGroupBatch;
- block.isInputExpressionEvaluated[i] = bufferedBatch.isInputExpressionEvaluated;
block.spillRowCount += bufferedBatch.size;
block.spillBatchCount += 1;
if (previousBatch != null) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
index 5accf6950c3..feb12617de1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
@@ -28,7 +28,6 @@
import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -390,6 +389,18 @@ public void process(Object row, int tag) throws HiveException {
}
}
+ /*
+ Evaluate input expressions on the original batch before it gets buffered.
+ This ensures that complex expressions read from the correct columns in
+ the original batch and write their results to scratch columns, which
+ are then buffered.
+ */
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ if (evaluator.inputVecExpr != null) {
+ evaluator.inputVecExpr.evaluate(batch);
+ }
+ }
+
if (isFirstPartition) {
isFirstPartition = false;
setCurrentPartition(batch);
@@ -477,39 +488,27 @@ private void initBufferedColumns() {
}
private void initExpressionColumns() {
- for (int i = 0; i < evaluators.length; i++) {
- VectorPTFEvaluatorBase evaluator = evaluators[i];
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
/*
- * Non-streaming evaluators work on buffered batches, we need to adapt them. Before PTF
- * bounded start vectorization (HIVE-24761), VectorExpression.outputColumnNum was closed and
- * VectorExpression.inputColumnNum didn't even exist (even though the vast majority of
- * VectorExpression subclasses use at least 1 input column). Since VectorPTFOperator and
- * VectorPTFGroupBatches work on modified batches (by not storing all the columns, and having
- * ordering columns first), the expressions planned in compile-time won't work with the
- * original config (column layout). It would make sense to move this logic to compile time,
- * because here in runtime, a very simple mapping (bufferedColumnMap) is used, so it might be
- * used. However, vectorized expression compilation affects many layers of code (having
- * VectorizationContext as the common scope), and moving the calculation of bufferedColumnMap
- * and this override logic to compile-time would create much more complicated behavior there
- * (probably involving hacking most of the time, or maybe a great re-design) just because of
- * the optimized column layout of the PTF vectorization.
+ * Non-streaming window function evaluators (like SUM or LEAD) work on packed, buffered batches.
+ * VectorPTFOperator and VectorPTFGroupBatches significantly modify the batch structure to save memory
+ * (e.g. by dropping unused columns and moving ordering columns to the front).
+ *
+ * Because of this dense packing, the absolute column indices assigned by the compiler (e.g., Index 15)
+ * are no longer valid for the buffer (where it might now be Index 2).
+ * Therefore, we must dynamically map the evaluator's inputColumnNum to its new location in the packed array.
+ *
+ * NOTE: We ONLY patch the evaluator's input index here. We specifically DO NOT patch
+ * evaluator.inputVecExpr (the math expressions like A+B) because those are now evaluated eagerly
+ * on the original batch before any packing occurs, so their absolute indices remain perfectly valid.
*/
if (!evaluator.streamsResult()) {
- evaluator.inputColumnNum = IntStream.range(0, bufferedColumnMap.length)
- .filter(j -> bufferedColumnMap[j] == evaluator.inputColumnNum).findFirst()
- .orElseGet(() -> evaluator.inputColumnNum);
-
- if (evaluator.inputVecExpr != null) {
- for (int j = 0; j < evaluator.inputVecExpr.inputColumnNum.length; j++){
- final int jj = j; // need a final in stream filter
- evaluator.inputVecExpr.inputColumnNum[jj] = IntStream.range(0, bufferedColumnMap.length)
- .filter(k -> bufferedColumnMap[k] == evaluator.inputVecExpr.inputColumnNum[jj])
- .findFirst().orElseGet(() -> evaluator.inputVecExpr.inputColumnNum[jj]);
- }
- evaluator.inputVecExpr.outputColumnNum = IntStream.range(0, bufferedColumnMap.length)
- .filter(j -> bufferedColumnMap[j] == evaluator.inputVecExpr.outputColumnNum)
- .findFirst().orElseGet(() -> evaluator.inputVecExpr.outputColumnNum);
- }
+ evaluator.inputColumnNum =
+ IntStream.range(0, bufferedColumnMap.length)
+ .filter(j -> bufferedColumnMap[j] == evaluator.inputColumnNum)
+ .findFirst()
+ .orElseGet(() -> evaluator.inputColumnNum);
+
evaluator.mapCustomColumns(bufferedColumnMap);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
index 8d21892b785..a5c6170324c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
@@ -139,10 +139,6 @@ class VectorSpillBlock {
*/
boolean[] isLastGroupBatch;
- /**
- * Stores the isInputExpressionEvaluated property for buffered/spilled batches.
- */
- boolean[] isInputExpressionEvaluated;
/**
* <pre>
* Deserializer needs to know which columns to put the deserialized values into. Basically,
@@ -165,7 +161,6 @@ class VectorSpillBlock {
startBatchIndex = blockIndex * blockSize;
startRowIndex = new int[blockSize];
isLastGroupBatch = new boolean[blockSize];
- isInputExpressionEvaluated = new boolean[blockSize];
}
void releaseRowBytesContainer() {
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
new file mode 100644
index 00000000000..823954565e9
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
@@ -0,0 +1,56 @@
+set hive.cli.print.header=true;
+set hive.vectorized.execution.enabled=true;
+set hive.vectorized.execution.reduce.enabled=true;
+set hive.vectorized.execution.ptf.enabled=true;
+
+CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
new file mode 100644
index 00000000000..cf12f629649
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
@@ -0,0 +1,434 @@
+PREHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_decimal
+PREHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: Lineage: repro_ptf_decimal.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_decimal
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string, 1:val1:decimal(20,2), 2:val2:bigint, 3:val3:smallint, 4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>, 5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type: decimal(20,2)), id (type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:decimal(20,2), 0:string
+ keyExpressions: ConstantVectorExpression(val 0) -> 6:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:bigint, 3:smallint
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: bigint), val3 (type: smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:decimal(20,2), val2:bigint, val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:decimal(20,2), KEY.reducesinkkey2:string, VALUE._col0:bigint, VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(34,2), bigint, decimal(22,2), decimal(19,0), decimal(23,2), decimal(5,0), decimal(24,2)]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: decimal(20,2)), VALUE._col0 (type: bigint), VALUE._col1 (type: smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: decimal(20,2), _col2: bigint, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * CAST( 2 AS decimal(10,0))) + CAST( _col2 AS decimal(19,0))) - CAST( _col3 AS decimal(5,0)))
+ name: sum
+ window function: GenericUDAFSumHiveDecimal
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDecimalSum]
+ functionInputExpressions: [DecimalColSubtractDecimalColumn(col 9:decimal(23,2), col 10:decimal(5,0))(children: DecimalColAddDecimalColumn(col 7:decimal(22,2), col 8:decimal(19,0))(children: DecimalColMultiplyDecimalScalar(col 1:decimal(20,2), val 2) -> 7:decimal(22,2), CastLongToDecimal(col 3:bigint) -> 8:decimal(19,0)) -> 9:decimal(23,2), CastLongToDecimal(col 4:smallint) -> 10:decimal(5,0)) -> 11:decimal(24,2)]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:decimal(20,2), col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [decimal(34,2), string, decimal(20,2), bigint, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: decimal(20,2)), _col2 (type: bigint), _col3 (type: smallint), sum_window_0 (type: decimal(34,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10.00 4 2 22.00
+req2 10.00 5 2 45.00
+req1 5.00 2 1 56.00
+PREHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_long
+PREHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_long
+POSTHOOK: Lineage: repro_ptf_long.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_long
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string, 1:val1:bigint, 2:val2:int, 3:val3:smallint, 4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>, 5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type: bigint), id (type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:bigint, 0:string
+ keyExpressions: ConstantVectorExpression(val 0) -> 6:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:int, 3:smallint
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: int), val3 (type: smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:bigint, val2:int, val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:bigint, KEY.reducesinkkey2:string, VALUE._col0:int, VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: bigint, _col2: int, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * UDFToLong(2)) + UDFToLong(_col2)) - UDFToLong(_col3))
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [LongColSubtractLongColumn(col 8:bigint, col 4:bigint)(children: LongColAddLongColumn(col 7:bigint, col 3:bigint)(children: LongColMultiplyLongScalar(col 1:bigint, val 2) -> 7:bigint, col 3:int) -> 8:bigint, col 4:smallint) -> 9:bigint]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:bigint, col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [bigint, string, bigint, int, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int), _col3 (type: smallint), sum_window_0 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10 4 2 22
+req2 10 5 2 45
+req1 5 2 1 56