You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/02/16 15:52:34 UTC
[20/32] hive git commit: HIVE-18622: Vectorization: IF Statements,
Comparisons, and more do not handle NULLs correctly (Matt McCline,
reviewed by Sergey Shelukhin, Deepak Jaiswal, Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_decimal_math_funcs.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_decimal_math_funcs.q b/ql/src/test/queries/clientpositive/vector_decimal_math_funcs.q
index ee9f333..4264937 100644
--- a/ql/src/test/queries/clientpositive/vector_decimal_math_funcs.q
+++ b/ql/src/test/queries/clientpositive/vector_decimal_math_funcs.q
@@ -4,6 +4,9 @@ set hive.fetch.task.conversion=none;
CREATE TABLE decimal_test STORED AS ORC AS SELECT cbigint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc;
+-- Add a single NULL row that will come from ORC as isRepeated.
+insert into decimal_test values (NULL, NULL, NULL, NULL);
+
SET hive.vectorized.execution.enabled=true;
set hive.fetch.task.conversion=none;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_decimal_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_decimal_udf.q b/ql/src/test/queries/clientpositive/vector_decimal_udf.q
index 13e5686..091f502 100644
--- a/ql/src/test/queries/clientpositive/vector_decimal_udf.q
+++ b/ql/src/test/queries/clientpositive/vector_decimal_udf.q
@@ -3,6 +3,8 @@ set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
set hive.fetch.task.conversion=minimal;
+-- SORT_QUERY_RESULTS
+
DROP TABLE IF EXISTS DECIMAL_UDF_txt;
DROP TABLE IF EXISTS DECIMAL_UDF;
@@ -18,6 +20,10 @@ STORED AS ORC;
INSERT OVERWRITE TABLE DECIMAL_UDF SELECT * FROM DECIMAL_UDF_txt;
+-- Add a single NULL row that will come from ORC as isRepeated.
+insert into DECIMAL_UDF values (NULL, NULL);
+
+
-- addition
EXPLAIN VECTORIZATION DETAIL
SELECT key + key FROM DECIMAL_UDF;
@@ -75,13 +81,13 @@ SELECT key * '2.0' FROM DECIMAL_UDF;
-- division
EXPLAIN VECTORIZATION DETAIL
-SELECT key / 0 FROM DECIMAL_UDF limit 1;
-SELECT key / 0 FROM DECIMAL_UDF limit 1;
+SELECT key / 0 FROM DECIMAL_UDF;
+SELECT key / 0 FROM DECIMAL_UDF;
-- Output not stable.
-- EXPLAIN VECTORIZATION DETAIL
--- SELECT key / NULL FROM DECIMAL_UDF limit 1;
--- SELECT key / NULL FROM DECIMAL_UDF limit 1;
+-- SELECT key / NULL FROM DECIMAL_UDF;
+-- SELECT key / NULL FROM DECIMAL_UDF;
EXPLAIN VECTORIZATION DETAIL
SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0;
@@ -182,6 +188,9 @@ STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH '../../data/files/kv7.txt' INTO TABLE DECIMAL_UDF_txt_small;
+-- Add a single NULL row.
+insert into DECIMAL_UDF_txt_small values (NULL, NULL);
+
-- addition
EXPLAIN VECTORIZATION DETAIL
SELECT key + key FROM DECIMAL_UDF_txt_small;
@@ -239,13 +248,13 @@ SELECT key * '2.0' FROM DECIMAL_UDF_txt_small;
-- division
EXPLAIN VECTORIZATION DETAIL
-SELECT key / 0 FROM DECIMAL_UDF_txt_small limit 1;
-SELECT key / 0 FROM DECIMAL_UDF_txt_small limit 1;
+SELECT key / 0 FROM DECIMAL_UDF_txt_small;
+SELECT key / 0 FROM DECIMAL_UDF_txt_small;
-- Output not stable.
-- EXPLAIN VECTORIZATION DETAIL
--- SELECT key / NULL FROM DECIMAL_UDF_txt_small limit 1;
--- SELECT key / NULL FROM DECIMAL_UDF_txt_small limit 1;
+-- SELECT key / NULL FROM DECIMAL_UDF_txt_small;
+-- SELECT key / NULL FROM DECIMAL_UDF_txt_small;
EXPLAIN VECTORIZATION DETAIL
SELECT key / key FROM DECIMAL_UDF_txt_small WHERE key is not null and key <> 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_decimal_udf2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_decimal_udf2.q b/ql/src/test/queries/clientpositive/vector_decimal_udf2.q
index 540fb7b..e10b7d1 100644
--- a/ql/src/test/queries/clientpositive/vector_decimal_udf2.q
+++ b/ql/src/test/queries/clientpositive/vector_decimal_udf2.q
@@ -18,6 +18,9 @@ STORED AS ORC;
INSERT OVERWRITE TABLE DECIMAL_UDF2 SELECT * FROM DECIMAL_UDF2_txt;
+-- Add a single NULL row that will come from ORC as isRepeated.
+insert into DECIMAL_UDF2 values (NULL, NULL);
+
EXPLAIN VECTORIZATION DETAIL
SELECT acos(key), asin(key), atan(key), cos(key), sin(key), tan(key), radians(key)
FROM DECIMAL_UDF2 WHERE key = 10;
@@ -25,6 +28,10 @@ FROM DECIMAL_UDF2 WHERE key = 10;
SELECT acos(key), asin(key), atan(key), cos(key), sin(key), tan(key), radians(key)
FROM DECIMAL_UDF2 WHERE key = 10;
+SELECT SUM(HASH(*))
+FROM (SELECT acos(key), asin(key), atan(key), cos(key), sin(key), tan(key), radians(key)
+FROM DECIMAL_UDF2) q;
+
EXPLAIN VECTORIZATION DETAIL
SELECT
exp(key), ln(key),
@@ -38,6 +45,13 @@ SELECT
log10(key), sqrt(key)
FROM DECIMAL_UDF2 WHERE key = 10;
+SELECT SUM(HASH(*))
+FROM (SELECT
+ exp(key), ln(key),
+ log(key), log(key, key), log(key, value), log(value, key),
+ log10(key), sqrt(key)
+FROM DECIMAL_UDF2) q;
+
-- DECIMAL_64
EXPLAIN VECTORIZATION DETAIL
@@ -47,6 +61,10 @@ FROM DECIMAL_UDF2_txt WHERE key = 10;
SELECT acos(key), asin(key), atan(key), cos(key), sin(key), tan(key), radians(key)
FROM DECIMAL_UDF2_txt WHERE key = 10;
+SELECT SUM(HASH(*))
+FROM (SELECT acos(key), asin(key), atan(key), cos(key), sin(key), tan(key), radians(key)
+FROM DECIMAL_UDF2_txt) q;
+
EXPLAIN VECTORIZATION DETAIL
SELECT
exp(key), ln(key),
@@ -60,5 +78,12 @@ SELECT
log10(key), sqrt(key)
FROM DECIMAL_UDF2_txt WHERE key = 10;
+SELECT SUM(HASH(*))
+FROM (SELECT
+ exp(key), ln(key),
+ log(key), log(key, key), log(key, value), log(value, key),
+ log10(key), sqrt(key)
+FROM DECIMAL_UDF2_txt) q;
+
DROP TABLE IF EXISTS DECIMAL_UDF2_txt;
DROP TABLE IF EXISTS DECIMAL_UDF2;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_interval_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_interval_1.q b/ql/src/test/queries/clientpositive/vector_interval_1.q
index f4f0024..3702734 100644
--- a/ql/src/test/queries/clientpositive/vector_interval_1.q
+++ b/ql/src/test/queries/clientpositive/vector_interval_1.q
@@ -2,6 +2,7 @@ set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
set hive.vectorized.execution.enabled=true;
set hive.fetch.task.conversion=none;
+set hive.cli.print.header=true;
drop table if exists vector_interval_1;
create table vector_interval_1 (ts timestamp, dt date, str1 string, str2 string) stored as orc;
@@ -11,6 +12,8 @@ insert into vector_interval_1
insert into vector_interval_1
select null, null, null, null from src limit 1;
+select * from vector_interval_1;
+
-- constants/cast from string
explain vectorization expression
select
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_ptf_part_simple.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_part_simple.q b/ql/src/test/queries/clientpositive/vector_ptf_part_simple.q
index fc9f9eb..5615bca 100644
--- a/ql/src/test/queries/clientpositive/vector_ptf_part_simple.q
+++ b/ql/src/test/queries/clientpositive/vector_ptf_part_simple.q
@@ -531,19 +531,37 @@ count(*) over(partition by p_mfgr order by p_name) as cs
from vector_ptf_part_simple_orc;
+explain vectorization detail
+select p_mfgr, p_retailprice,
+rank() over(partition by p_mfgr) as r
+from vector_ptf_part_simple_orc;
+
select p_mfgr, p_retailprice,
rank() over(partition by p_mfgr) as r
from vector_ptf_part_simple_orc;
+explain vectorization detail
+select p_mfgr, p_retailprice,
+rank() over(partition by p_mfgr order by p_name) as r
+from vector_ptf_part_simple_orc;
+
select p_mfgr, p_retailprice,
rank() over(partition by p_mfgr order by p_name) as r
from vector_ptf_part_simple_orc;
+explain vectorization detail
+select p_mfgr, p_name, p_retailprice,
+rank() over(partition by p_mfgr, case when p_mfgr == "Manufacturer#2" then timestamp "2000-01-01 00:00:00" end order by p_name) as r
+from vector_ptf_part_simple_orc;
select p_mfgr, p_name, p_retailprice,
rank() over(partition by p_mfgr, case when p_mfgr == "Manufacturer#2" then timestamp "2000-01-01 00:00:00" end order by p_name) as r
from vector_ptf_part_simple_orc;
+explain vectorization detail
+select p_mfgr, p_name, p_retailprice,
+rank() over(partition by p_mfgr, case when p_mfgr == "Manufacturer#2" then timestamp "2000-01-01 00:00:00" end) as r
+from vector_ptf_part_simple_orc;
select p_mfgr, p_name, p_retailprice,
rank() over(partition by p_mfgr, case when p_mfgr == "Manufacturer#2" then timestamp "2000-01-01 00:00:00" end) as r
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_udf2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_udf2.q b/ql/src/test/queries/clientpositive/vector_udf2.q
index e62af6a..bd8e875 100644
--- a/ql/src/test/queries/clientpositive/vector_udf2.q
+++ b/ql/src/test/queries/clientpositive/vector_udf2.q
@@ -33,10 +33,16 @@ create temporary table HIVE_14349 (a string) stored as orc;
insert into HIVE_14349 values('XYZa'), ('badXYZa');
+explain vectorization expression
+select * from HIVE_14349 where a LIKE 'XYZ%a%';
+
select * from HIVE_14349 where a LIKE 'XYZ%a%';
insert into HIVE_14349 values ('XYZab'), ('XYZabBAD'), ('badXYZab'), ('badXYZabc');
+explain vectorization expression
+select * from HIVE_14349 where a LIKE 'XYZ%a_';
+
select * from HIVE_14349 where a LIKE 'XYZ%a_';
drop table HIVE_14349;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vector_udf_string_to_boolean.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_udf_string_to_boolean.q b/ql/src/test/queries/clientpositive/vector_udf_string_to_boolean.q
index eeb5ab8..5c052a1 100644
--- a/ql/src/test/queries/clientpositive/vector_udf_string_to_boolean.q
+++ b/ql/src/test/queries/clientpositive/vector_udf_string_to_boolean.q
@@ -1,5 +1,5 @@
set hive.mapred.mode=nonstrict;
-SET hive.vectorized.execution.enabled = true;
+SET hive.vectorized.execution.enabled = false;
SET hive.int.timestamp.conversion.in.seconds=false;
set hive.fetch.task.conversion=none;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vectorization_div0.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorization_div0.q b/ql/src/test/queries/clientpositive/vectorization_div0.q
index d7b6c3c..26ff266 100644
--- a/ql/src/test/queries/clientpositive/vectorization_div0.q
+++ b/ql/src/test/queries/clientpositive/vectorization_div0.q
@@ -3,6 +3,8 @@ set hive.explain.user=false;
SET hive.vectorized.execution.enabled = true;
set hive.fetch.task.conversion=none;
+-- SORT_QUERY_RESULTS
+
-- TODO: add more stuff here after HIVE-5918 is fixed, such as cbigint and constants
explain vectorization expression
select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorization_nested_udf.q b/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
index da8f99c..25a25df 100644
--- a/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
+++ b/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
@@ -1,5 +1,8 @@
+set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
set hive.fetch.task.conversion=none;
+EXPLAIN VECTORIZATION DETAIL
+SELECT SUM(abs(ctinyint)) from alltypesorc;
SELECT SUM(abs(ctinyint)) from alltypesorc;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/queries/clientpositive/vectorized_case.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_case.q b/ql/src/test/queries/clientpositive/vectorized_case.q
index 99d7cfc..8aad2b5 100644
--- a/ql/src/test/queries/clientpositive/vectorized_case.q
+++ b/ql/src/test/queries/clientpositive/vectorized_case.q
@@ -1,8 +1,8 @@
set hive.explain.user=false;
set hive.fetch.task.conversion=none;
-set hive.vectorized.execution.enabled = true
-;
-explain vectorization expression
+set hive.vectorized.execution.enabled = true;
+
+explain vectorization detail
select
csmallint,
case
@@ -37,7 +37,7 @@ where csmallint = 418
or csmallint = 12205
or csmallint = 10583
;
-explain vectorization expression
+explain vectorization detail
select
csmallint,
case
@@ -55,7 +55,7 @@ where csmallint = 418
or csmallint = 12205
or csmallint = 10583
;
-explain vectorization expression
+explain vectorization detail
select
sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
sum(case when cint % 2 = 1 then 1 else 0 end) as codd
@@ -64,7 +64,7 @@ select
sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
sum(case when cint % 2 = 1 then 1 else 0 end) as codd
from alltypesorc;
-explain vectorization expression
+explain vectorization detail
select
sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
sum(case when cint % 2 = 1 then cint else 0 end) as codd
@@ -79,19 +79,19 @@ CREATE TABLE test_1 (member DECIMAL , attr DECIMAL) STORED AS ORC;
INSERT INTO test_1 VALUES (3.0,1.0),(2.0,2.0),(1.0,3.0);
--for length=3
-EXPLAIN VECTORIZATION EXPRESSION
+EXPLAIN VECTORIZATION DETAIL
SELECT CASE WHEN member =1.0 THEN attr+1.0 ELSE attr+2.0 END FROM test_1;
SELECT CASE WHEN member =1.0 THEN attr+1.0 ELSE attr+2.0 END FROM test_1;
--for length=2 and the expr2 is null
-EXPLAIN VECTORIZATION EXPRESSION
+EXPLAIN VECTORIZATION DETAIL
SELECT CASE WHEN member =1.0 THEN 1.0 ELSE attr+2.0 END FROM test_1;
SELECT CASE WHEN member =1.0 THEN 1.0 ELSE attr+2.0 END FROM test_1;
--for length=2 and the expr3 is null
-EXPLAIN VECTORIZATION EXPRESSION
+EXPLAIN VECTORIZATION DETAIL
SELECT CASE WHEN member =1.0 THEN attr+1.0 ELSE 2.0 END FROM test_1;
SELECT CASE WHEN member =1.0 THEN attr+1.0 ELSE 2.0 END FROM test_1;
@@ -102,19 +102,105 @@ CREATE TABLE test_2 (member BIGINT, attr BIGINT) STORED AS ORC;
INSERT INTO test_2 VALUES (3,1),(2,2),(1,3);
--for length=3
-EXPLAIN VECTORIZATION EXPRESSION
+EXPLAIN VECTORIZATION DETAIL
SELECT CASE WHEN member=1 THEN attr+1 else attr+2 END FROM test_2;
SELECT CASE WHEN member=1 THEN attr+1 else attr+2 END FROM test_2;
---for length=2 and the expression2 is null
-EXPLAIN VECTORIZATION EXPRESSION
+--for length=2 and the detail2 is null
+EXPLAIN VECTORIZATION DETAIL
SELECT CASE WHEN member=1 THEN null else attr+2 END FROM test_2;
SELECT CASE WHEN member=1 THEN null else attr+2 END FROM test_2;
---for length=2 and the expression3 is null
-EXPLAIN VECTORIZATION EXPRESSION
+--for length=2 and the detail3 is null
+EXPLAIN VECTORIZATION DETAIL
+SELECT CASE WHEN member=1 THEN attr+1 else null END FROM test_2;
+
SELECT CASE WHEN member=1 THEN attr+1 else null END FROM test_2;
-SELECT CASE WHEN member=1 THEN attr+1 else null END FROM test_2;
\ No newline at end of file
+
+select count(*), sum(a.ceven)
+from (
+select
+ case when cint % 2 = 0 then 1 else 0 end as ceven
+from alltypesorc) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then 1 else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then 1 else 0 end) = 0) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then 1 else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then 1 else 0 end) = 0 AND cint is NOT NULL) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then 1 else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then 1 else 0 end) = 1) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then 1 else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then 1 else 0 end) = 1 AND cint is NOT NULL) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then 1 else 0 end) as ceven
+from alltypesorc
+where cint is null) a;
+
+
+select count(*), sum(a.ceven)
+from (
+select
+ case when cint % 2 = 0 then cint else 0 end as ceven
+from alltypesorc) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then cint else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then cint else 0 end) = 0) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then cint else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then cint else 0 end) = 0 AND cint is NOT NULL) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then cint else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then cint else 0 end) = cint) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then cint else 0 end) as ceven
+from alltypesorc
+where (case when cint % 2 = 0 then cint else 0 end) = cint AND cint is NOT NULL) a;
+
+select count(*)
+from (
+select
+ (case when cint % 2 = 0 then cint else 0 end) as ceven
+from alltypesorc
+where cint is null) a;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
index 9ce5e1e..d77684b 100644
--- a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
@@ -24,6 +24,18 @@ POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:
POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert into varchar_udf_1 values (NULL, NULL, NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@varchar_udf_1
+POSTHOOK: query: insert into varchar_udf_1 values (NULL, NULL, NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@varchar_udf_1
+POSTHOOK: Lineage: varchar_udf_1.c1 EXPRESSION []
+POSTHOOK: Lineage: varchar_udf_1.c2 EXPRESSION []
+POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION []
+POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION []
PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF_txt
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF_txt
@@ -74,6 +86,16 @@ POSTHOOK: Input: default@decimal_udf_txt
POSTHOOK: Output: default@decimal_udf
POSTHOOK: Lineage: decimal_udf.key SIMPLE [(decimal_udf_txt)decimal_udf_txt.FieldSchema(name:key, type:decimal(20,10), comment:null), ]
POSTHOOK: Lineage: decimal_udf.value SIMPLE [(decimal_udf_txt)decimal_udf_txt.FieldSchema(name:value, type:int, comment:null), ]
+PREHOOK: query: insert into DECIMAL_UDF values (NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@decimal_udf
+POSTHOOK: query: insert into DECIMAL_UDF values (NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@decimal_udf
+POSTHOOK: Lineage: decimal_udf.key EXPRESSION []
+POSTHOOK: Lineage: decimal_udf.value EXPRESSION []
PREHOOK: query: drop table if exists count_case_groupby
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table if exists count_case_groupby
@@ -96,23 +118,33 @@ POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@count_case_groupby
POSTHOOK: Lineage: count_case_groupby.bool SCRIPT []
POSTHOOK: Lineage: count_case_groupby.key SCRIPT []
+PREHOOK: query: insert into table count_case_groupby values (NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@count_case_groupby
+POSTHOOK: query: insert into table count_case_groupby values (NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@count_case_groupby
+POSTHOOK: Lineage: count_case_groupby.bool EXPRESSION []
+POSTHOOK: Lineage: count_case_groupby.key EXPRESSION []
PREHOOK: query: explain vectorization expression
select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -127,33 +159,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: c2 regexp 'val' (type: boolean), c4 regexp 'val' (type: boolean), (c2 regexp 'val' = c4 regexp 'val') (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFRegExp(Column[c2], Const string val) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -161,7 +184,7 @@ PREHOOK: query: select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -169,28 +192,29 @@ POSTHOOK: query: select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
+NULL NULL NULL
true true true
PREHOOK: query: explain vectorization expression
select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -205,33 +229,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: regexp_extract(c2, 'val_([0-9]+)', 1) (type: string), regexp_extract(c4, 'val_([0-9]+)', 1) (type: string), (regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)) (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFBridge ==> regexp_extract (Column[c2], Const string val_([0-9]+), Const int 1) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -239,7 +254,7 @@ PREHOOK: query: select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -247,28 +262,29 @@ POSTHOOK: query: select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
238 238 true
+NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -283,33 +299,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: regexp_replace(c2, 'val', 'replaced') (type: string), regexp_replace(c4, 'val', 'replaced') (type: string), (regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')) (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFBridge ==> regexp_replace (Column[c2], Const string val, Const string replaced) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -317,7 +324,7 @@ PREHOOK: query: select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -325,28 +332,29 @@ POSTHOOK: query: select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
+NULL NULL NULL
replaced_238 replaced_238 true
PREHOOK: query: explain vectorization expression
select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -361,33 +369,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: c2 regexp 'val' (type: boolean), c4 regexp 'val' (type: boolean), (c2 regexp 'val' = c4 regexp 'val') (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFRegExp(Column[c2], Const string val) because hive.vectorized.adaptor.usage.mode=chosen and the UDF wasn't one of the chosen ones
- vectorized: false
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -395,7 +394,7 @@ PREHOOK: query: select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -403,28 +402,29 @@ POSTHOOK: query: select
c2 regexp 'val',
c4 regexp 'val',
(c2 regexp 'val') = (c4 regexp 'val')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
+NULL NULL NULL
true true true
PREHOOK: query: explain vectorization expression
select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -439,49 +439,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan Vectorization:
- native: true
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: regexp_extract(c2, 'val_([0-9]+)', 1) (type: string), regexp_extract(c4, 'val_([0-9]+)', 1) (type: string), (regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)) (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [5, 6, 9]
- selectExpressions: VectorUDFAdaptor(regexp_extract(c2, 'val_([0-9]+)', 1)) -> 5:string, VectorUDFAdaptor(regexp_extract(c4, 'val_([0-9]+)', 1)) -> 6:string, StringGroupColEqualStringGroupColumn(col 7:string, col 8:string)(children: VectorUDFAdaptor(regexp_extract(c2, 'val_([0-9]+)', 1)) -> 7:string, VectorUDFAdaptor(regexp_extract(c4, 'val_([0-9]+)', 1)) -> 8:string) -> 9:boolean
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Limit Vectorization:
- className: VectorLimitOperator
- native: true
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized, llap
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: true
- vectorized: true
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -489,7 +464,7 @@ PREHOOK: query: select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -497,28 +472,29 @@ POSTHOOK: query: select
regexp_extract(c2, 'val_([0-9]+)', 1),
regexp_extract(c4, 'val_([0-9]+)', 1),
regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1)
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
238 238 true
+NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -533,49 +509,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: varchar_udf_1
- Statistics: Num rows: 1 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan Vectorization:
- native: true
+ Statistics: Num rows: 2 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: regexp_replace(c2, 'val', 'replaced') (type: string), regexp_replace(c4, 'val', 'replaced') (type: string), (regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')) (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [5, 6, 9]
- selectExpressions: VectorUDFAdaptor(regexp_replace(c2, 'val', 'replaced')) -> 5:string, VectorUDFAdaptor(regexp_replace(c4, 'val', 'replaced')) -> 6:string, StringGroupColEqualStringGroupColumn(col 7:string, col 8:string)(children: VectorUDFAdaptor(regexp_replace(c2, 'val', 'replaced')) -> 7:string, VectorUDFAdaptor(regexp_replace(c4, 'val', 'replaced')) -> 8:string) -> 9:boolean
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Limit Vectorization:
- className: VectorLimitOperator
- native: true
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized, llap
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 744 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: true
- vectorized: true
Stage: Stage-0
Fetch Operator
- limit: 1
+ limit: -1
Processor Tree:
ListSink
@@ -583,7 +534,7 @@ PREHOOK: query: select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
PREHOOK: type: QUERY
PREHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
@@ -591,18 +542,19 @@ POSTHOOK: query: select
regexp_replace(c2, 'val', 'replaced'),
regexp_replace(c4, 'val', 'replaced'),
regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced')
-from varchar_udf_1 limit 1
+from varchar_udf_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@varchar_udf_1
#### A masked pattern was here ####
+NULL NULL NULL
replaced_238 replaced_238 true
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT POWER(key, 2) FROM DECIMAL_UDF
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT POWER(key, 2) FROM DECIMAL_UDF
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -617,26 +569,20 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_udf
- Statistics: Num rows: 38 Data size: 4256 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 4256 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: power(key, 2) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFPower(Column[key], Const int 2) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
Stage: Stage-0
Fetch Operator
@@ -690,6 +636,7 @@ POSTHOOK: Input: default@decimal_udf
9.8596
9.8596
NULL
+NULL
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT
exp(key), ln(key),
@@ -705,8 +652,8 @@ SELECT
FROM DECIMAL_UDF WHERE key = 10
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -721,7 +668,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_udf
- Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 4412 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key = 10) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
@@ -738,12 +685,6 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFBridge ==> log (Column[value], Const decimal(20,10) 10) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
Stage: Stage-0
Fetch Operator
@@ -773,8 +714,8 @@ PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT POWER(key, 2) FROM DECIMAL_UDF
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -789,26 +730,20 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_udf
- Statistics: Num rows: 38 Data size: 4256 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 4256 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: power(key, 2) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 304 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFPower(Column[key], Const int 2) because hive.vectorized.adaptor.usage.mode=chosen and the UDF wasn't one of the chosen ones
- vectorized: false
Stage: Stage-0
Fetch Operator
@@ -862,6 +797,7 @@ POSTHOOK: Input: default@decimal_udf
9.8596
9.8596
NULL
+NULL
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT
exp(key), ln(key),
@@ -877,8 +813,8 @@ SELECT
FROM DECIMAL_UDF WHERE key = 10
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -893,7 +829,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_udf
- Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 39 Data size: 4412 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key = 10) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
@@ -910,12 +846,6 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFBridge ==> log (Column[value], Const decimal(20,10) 10) because hive.vectorized.adaptor.usage.mode=chosen and the UDF wasn't one of the chosen ones
- vectorized: false
Stage: Stage-0
Fetch Operator
@@ -947,8 +877,8 @@ POSTHOOK: query: explain vectorization expression
SELECT key, COUNT(CASE WHEN bool THEN 1 WHEN NOT bool THEN 0 ELSE NULL END) AS cnt_bool0_ok FROM count_case_groupby GROUP BY key
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -966,82 +896,37 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: count_case_groupby
- Statistics: Num rows: 5 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan Vectorization:
- native: true
+ Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END (type: int)
outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 6]
- selectExpressions: IfExprLongScalarLongColumn(col 1:boolean, val 1, col 5:int)(children: IfExprColumnNull(col 3:boolean, col 4:int, null)(children: NotCol(col 1:boolean) -> 3:boolean, ConstantVectorExpression(val 0) -> 4:int) -> 5:int) -> 6:int
- Statistics: Num rows: 5 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col1)
- Group By Vectorization:
- aggregators: VectorUDAFCount(col 6:int) -> bigint
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: col 0:string
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Reduce Sink Vectorization:
- className: VectorReduceSinkStringOperator
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Execution mode: vectorized, llap
+ Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
Reducer 2
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- keyExpressions: col 0:string
- native: false
- vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumnNums: [0]
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1061,6 +946,7 @@ POSTHOOK: query: SELECT key, COUNT(CASE WHEN bool THEN 1 WHEN NOT bool THEN 0 EL
POSTHOOK: type: QUERY
POSTHOOK: Input: default@count_case_groupby
#### A masked pattern was here ####
+NULL 0
key1 1
key2 1
key3 0
@@ -1073,8 +959,8 @@ POSTHOOK: query: explain vectorization expression
SELECT key, COUNT(CASE WHEN bool THEN 1 WHEN NOT bool THEN 0 ELSE NULL END) AS cnt_bool0_ok FROM count_case_groupby GROUP BY key
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+ enabled: false
+ enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false]
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1092,82 +978,37 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: count_case_groupby
- Statistics: Num rows: 5 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan Vectorization:
- native: true
+ Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END (type: int)
outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 6]
- selectExpressions: IfExprLongScalarLongColumn(col 1:boolean, val 1, col 5:int)(children: IfExprColumnNull(col 3:boolean, col 4:int, null)(children: NotCol(col 1:boolean) -> 3:boolean, ConstantVectorExpression(val 0) -> 4:int) -> 5:int) -> 6:int
- Statistics: Num rows: 5 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col1)
- Group By Vectorization:
- aggregators: VectorUDAFCount(col 6:int) -> bigint
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: col 0:string
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Reduce Sink Vectorization:
- className: VectorReduceSinkStringOperator
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Execution mode: vectorized, llap
+ Execution mode: llap
LLAP IO: all inputs
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
Reducer 2
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- keyExpressions: col 0:string
- native: false
- vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumnNums: [0]
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1187,6 +1028,7 @@ POSTHOOK: query: SELECT key, COUNT(CASE WHEN bool THEN 1 WHEN NOT bool THEN 0 EL
POSTHOOK: type: QUERY
POSTHOOK: Input: default@count_case_groupby
#### A masked pattern was here ####
+NULL 0
key1 1
key2 1
key3 0
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
index 48d5275..c85c59e 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
@@ -47,6 +47,17 @@ POSTHOOK: Output: default@TSINT
POSTHOOK: Lineage: tsint.csint SIMPLE [(tsint_txt)tsint_txt.FieldSchema(name:csint, type:smallint, comment:null), ]
POSTHOOK: Lineage: tsint.rnum SIMPLE [(tsint_txt)tsint_txt.FieldSchema(name:rnum, type:int, comment:null), ]
tsint_txt.rnum tsint_txt.csint
+PREHOOK: query: insert into TSINT values (NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tsint
+POSTHOOK: query: insert into TSINT values (NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tsint
+POSTHOOK: Lineage: tsint.csint EXPRESSION []
+POSTHOOK: Lineage: tsint.rnum EXPRESSION []
+_col0 _col1
PREHOOK: query: create table TINT stored as orc AS SELECT * FROM TINT_txt
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@tint_txt
@@ -60,6 +71,17 @@ POSTHOOK: Output: default@TINT
POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ]
POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ]
tint_txt.rnum tint_txt.cint
+PREHOOK: query: insert into TINT values (NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tint
+POSTHOOK: query: insert into TINT values (NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tint
+POSTHOOK: Lineage: tint.cint EXPRESSION []
+POSTHOOK: Lineage: tint.rnum EXPRESSION []
+_col0 _col1
Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain vectorization expression
select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint
@@ -88,7 +110,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tint
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -98,14 +120,14 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: int)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -122,7 +144,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tsint
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -132,14 +154,14 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: smallint)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -162,14 +184,14 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 25 Data size: 425 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 36 Data size: 612 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col2 (type: int), _col1 (type: int), _col3 (type: smallint), CASE WHEN (_col1 BETWEEN UDFToInteger(_col3) AND UDFToInteger(_col3)) THEN ('Ok') ELSE ('NoOk') END (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 25 Data size: 425 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 36 Data size: 612 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 25 Data size: 425 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 36 Data size: 612 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -198,26 +220,37 @@ tint.rnum tsint.rnum tint.cint tsint.csint between_col
0 2 NULL 0 NoOk
0 3 NULL 1 NoOk
0 4 NULL 10 NoOk
+0 NULL NULL NULL NoOk
1 0 -1 NULL NoOk
1 1 -1 -1 Ok
1 2 -1 0 NoOk
1 3 -1 1 NoOk
1 4 -1 10 NoOk
+1 NULL -1 NULL NoOk
2 0 0 NULL NoOk
2 1 0 -1 NoOk
2 2 0 0 Ok
2 3 0 1 NoOk
2 4 0 10 NoOk
+2 NULL 0 NULL NoOk
3 0 1 NULL NoOk
3 1 1 -1 NoOk
3 2 1 0 NoOk
3 3 1 1 Ok
3 4 1 10 NoOk
+3 NULL 1 NULL NoOk
4 0 10 NULL NoOk
4 1 10 -1 NoOk
4 2 10 0 NoOk
4 3 10 1 NoOk
4 4 10 10 Ok
+4 NULL 10 NULL NoOk
+NULL 0 NULL NULL NoOk
+NULL 1 NULL -1 NoOk
+NULL 2 NULL 0 NoOk
+NULL 3 NULL 1 NoOk
+NULL 4 NULL 10 NoOk
+NULL NULL NULL NULL NoOk
Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain vectorization expression
select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint
@@ -246,7 +279,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tint
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -256,14 +289,14 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: int)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -280,7 +313,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tsint
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -290,14 +323,14 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: smallint)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -320,17 +353,17 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 25 Data size: 425 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 36 Data size: 612 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col1 BETWEEN UDFToInteger(_col3) AND UDFToInteger(_col3) (type: boolean)
- Statistics: Num rows: 2 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 68 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col2 (type: int), _col1 (type: int), _col3 (type: smallint)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 68 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 68 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat