You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2021/11/08 06:54:05 UTC
[hive] branch master updated: HIVE-25653: Incorrect results
returned by STDDEV, STDDEV_SAMP,
STDDEV_POP for floating point data types (Ashish Sharma,
reviewed by Adesh Rao, Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d0f77cc HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan)
d0f77cc is described below
commit d0f77cca1a6612894837a174440a5fd929cd3bcb
Author: Ashish Kumar Sharma <as...@gmail.com>
AuthorDate: Mon Nov 8 12:23:55 2021 +0530
HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
Closes (#2760)
---
.../hadoop/hive/ql/udf/generic/GenericUDAFStd.java | 8 +-
.../hive/ql/udf/generic/GenericUDAFVariance.java | 29 ++++--
ql/src/test/queries/clientpositive/stddev.q | 14 +++
.../clientpositive/llap/cbo_rp_windowing_2.q.out | 42 ++++-----
.../test/results/clientpositive/llap/stddev.q.out | 102 +++++++++++++++++++++
.../clientpositive/llap/vector_windowing.q.out | 42 ++++-----
.../results/clientpositive/llap/windowing.q.out | 42 ++++-----
7 files changed, 205 insertions(+), 74 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java
index 79b519c..729455c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import java.math.BigDecimal;
+import java.math.MathContext;
+
/**
* Compute the standard deviation by extending GenericUDAFVariance and
* overriding the terminate() method of the evaluator.
@@ -90,7 +93,10 @@ public class GenericUDAFStd extends GenericUDAFVariance {
* use it, etc.
*/
public static double calculateStdResult(double variance, long count) {
- return Math.sqrt(variance / count);
+ // TODO: BigDecimal.sqrt() is introduced in java 9. So change the below calculation once hive upgraded to java 9 or above.
+ BigDecimal bvariance = new BigDecimal(variance);
+ BigDecimal result = bvariance.divide(new BigDecimal(count), MathContext.DECIMAL128);
+ return Math.sqrt(result.doubleValue());
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java
index bb55d88..5e60edc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
+import java.math.BigDecimal;
+import java.math.MathContext;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -106,9 +108,14 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver {
*/
public static double calculateIntermediate(
long count, double sum, double value, double variance) {
- double t = count * value - sum;
- variance += (t * t) / ((double) count * (count - 1));
- return variance;
+ BigDecimal bcount,bsum,bvalue,bvariance;
+ bvariance = new BigDecimal(variance);
+ bsum = new BigDecimal(sum);
+ bvalue = new BigDecimal(value);
+ bcount = new BigDecimal(count);
+ BigDecimal t = bcount.multiply(bvalue).subtract(bsum);
+ bvariance = bvariance.add(t.multiply(t).divide(bcount.multiply(bcount.subtract(BigDecimal.ONE)),MathContext.DECIMAL128));
+ return bvariance.doubleValue();
}
/*
@@ -120,14 +127,16 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver {
long partialCount, long mergeCount, double partialSum, double mergeSum,
double partialVariance, double mergeVariance) {
- final double doublePartialCount = (double) partialCount;
- final double doubleMergeCount = (double) mergeCount;
+ final BigDecimal bPartialCount = new BigDecimal(partialCount);
+ final BigDecimal bMergeCount = new BigDecimal(mergeCount);
+ BigDecimal bmergeVariance = new BigDecimal(mergeVariance);
- double t = (doublePartialCount / doubleMergeCount) * mergeSum - partialSum;
- mergeVariance +=
- partialVariance + ((doubleMergeCount / doublePartialCount) /
- (doubleMergeCount + doublePartialCount)) * t * t;
- return mergeVariance;
+ BigDecimal t =
+ bPartialCount.divide(bMergeCount, MathContext.DECIMAL128).multiply(new BigDecimal(mergeSum)).subtract(new BigDecimal(partialSum));
+
+ bmergeVariance = bmergeVariance.add(new BigDecimal(partialVariance).add(
+ (bMergeCount.divide(bPartialCount,MathContext.DECIMAL128).divide(bMergeCount.add(bPartialCount),MathContext.DECIMAL128)).multiply(t).multiply(t)));
+ return bmergeVariance.doubleValue();
}
/*
diff --git a/ql/src/test/queries/clientpositive/stddev.q b/ql/src/test/queries/clientpositive/stddev.q
new file mode 100644
index 0000000..5f3d9ed
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stddev.q
@@ -0,0 +1,14 @@
+create table test ( col1 decimal(10,3) );
+insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72);
+select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test;
+drop table test;
+
+create table testpoint ( col1 decimal(10,3));
+insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445);
+select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint;
+drop table testpoint;
+
+create table testint(col1 int);
+insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89);
+select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint;
+drop table testint;
diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
index 6ec2d60..4146bc0 100644
--- a/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
@@ -633,9 +633,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334
Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6
Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14
Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14
-Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14
Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2
Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17
Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17
Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17
@@ -862,31 +862,31 @@ window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 precedi
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995
-Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221
-Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864
-Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007
+Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995
+Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221
+Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007
Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004
-Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502
-Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658
-Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995
-Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995
-Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676
-Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502
+Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658
+Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995
+Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995
+Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676
+Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446
Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003
Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664
-Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993
+Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993
Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111
-Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779
-Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125
+Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779
+Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125
Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995
-Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975
+Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975
Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881
-Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353
-Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999
-Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986
-Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002
+Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353
+Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999
+Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002
Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666
PREHOOK: query: select p_mfgr,p_name, p_size,
histogram_numeric(p_retailprice, 5) over w1 as hist,
diff --git a/ql/src/test/results/clientpositive/llap/stddev.q.out b/ql/src/test/results/clientpositive/llap/stddev.q.out
new file mode 100644
index 0000000..69d462a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/stddev.q.out
@@ -0,0 +1,102 @@
+PREHOOK: query: create table test ( col1 decimal(10,3) )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: create table test ( col1 decimal(10,3) )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test
+POSTHOOK: query: insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test
+POSTHOOK: Lineage: test.col1 SCRIPT []
+PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+#### A masked pattern was here ####
+POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+#### A masked pattern was here ####
+0.0 0.0 0.0 0.0 0.0 0.0
+PREHOOK: query: drop table test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@test
+PREHOOK: Output: default@test
+POSTHOOK: query: drop table test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@test
+POSTHOOK: Output: default@test
+PREHOOK: query: create table testpoint ( col1 decimal(10,3))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testpoint
+POSTHOOK: query: create table testpoint ( col1 decimal(10,3))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testpoint
+PREHOOK: query: insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@testpoint
+POSTHOOK: query: insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@testpoint
+POSTHOOK: Lineage: testpoint.col1 SCRIPT []
+PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testpoint
+#### A masked pattern was here ####
+POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testpoint
+#### A masked pattern was here ####
+0.33922629053550113 0.31406251814905023 0.31406251814905023 0.0986352653061225 0.0986352653061225 0.11507447619047624
+PREHOOK: query: drop table testpoint
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@testpoint
+PREHOOK: Output: default@testpoint
+POSTHOOK: query: drop table testpoint
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@testpoint
+POSTHOOK: Output: default@testpoint
+PREHOOK: query: create table testint(col1 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testint
+POSTHOOK: query: create table testint(col1 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testint
+PREHOOK: query: insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@testint
+POSTHOOK: query: insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@testint
+POSTHOOK: Lineage: testint.col1 SCRIPT []
+PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testint
+#### A masked pattern was here ####
+POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testint
+#### A masked pattern was here ####
+9.003702941938208 8.698403429493386 8.698403429493386 75.66222222222228 75.66222222222228 81.06666666666673
+PREHOOK: query: drop table testint
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@testint
+PREHOOK: Output: default@testint
+POSTHOOK: query: drop table testint
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@testint
+POSTHOOK: Output: default@testint
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index f33c28f..3d6e9ce 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -3440,9 +3440,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334
Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6
Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14
Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14
-Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14
Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2
Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17
Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17
Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17
@@ -4671,31 +4671,31 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
p_mfgr p_name p_size sdev sdev_pop uniq_size var cor covarp
-Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995
-Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221
-Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864
-Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007
+Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995
+Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221
+Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007
Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004
-Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502
-Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658
-Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995
-Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995
-Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676
-Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502
+Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658
+Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995
+Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995
+Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676
+Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446
Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003
Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664
-Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993
+Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993
Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111
-Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779
-Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125
+Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779
+Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125
Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995
-Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975
+Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975
Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881
-Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353
-Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999
-Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986
-Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002
+Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353
+Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999
+Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002
Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666
PREHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_size,
diff --git a/ql/src/test/results/clientpositive/llap/windowing.q.out b/ql/src/test/results/clientpositive/llap/windowing.q.out
index 8c1f93a..c7415e7 100644
--- a/ql/src/test/results/clientpositive/llap/windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/windowing.q.out
@@ -633,9 +633,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334
Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6
Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14
Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14
-Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14
Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2
Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17
Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17
Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17
@@ -862,31 +862,31 @@ window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 precedi
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995
-Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221
-Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864
-Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007
+Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995
+Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221
+Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007
Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004
-Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502
-Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658
-Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995
-Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995
-Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676
-Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502
+Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658
+Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995
+Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995
+Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676
+Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446
Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003
Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664
-Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993
+Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993
Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111
-Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779
-Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125
+Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779
+Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125
Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995
-Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975
+Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975
Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881
-Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353
-Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999
-Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986
-Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002
+Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353
+Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999
+Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002
Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666
PREHOOK: query: select p_mfgr,p_name, p_size,
histogram_numeric(p_retailprice, 5) over w1 as hist,