You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2017/02/02 03:10:45 UTC
hive git commit: HIVE-14420: Fix orc_llap_counters.q test failure in
master (Prasanth Jayachandran reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/master 3f5f3ed62 -> 7cca0978a
HIVE-14420: Fix orc_llap_counters.q test failure in master (Prasanth Jayachandran reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7cca0978
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7cca0978
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7cca0978
Branch: refs/heads/master
Commit: 7cca0978af944b4a76dd40e014e628f82f43c42f
Parents: 3f5f3ed
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Wed Feb 1 19:10:28 2017 -0800
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Wed Feb 1 19:10:28 2017 -0800
----------------------------------------------------------------------
.../queries/clientpositive/orc_llap_counters.q | 12 +--
.../queries/clientpositive/orc_llap_counters1.q | 12 +--
.../clientpositive/llap/orc_llap_counters.q.out | 97 +++++++++-----------
.../llap/orc_llap_counters1.q.out | 37 +++-----
4 files changed, 70 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7cca0978/ql/src/test/queries/clientpositive/orc_llap_counters.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters.q b/ql/src/test/queries/clientpositive/orc_llap_counters.q
index cc0e991..2d8e3b1 100644
--- a/ql/src/test/queries/clientpositive/orc_llap_counters.q
+++ b/ql/src/test/queries/clientpositive/orc_llap_counters.q
@@ -4,6 +4,8 @@ SET hive.optimize.index.filter=true;
SET hive.cbo.enable=false;
SET hive.vectorized.execution.enabled=true;
SET hive.llap.io.enabled=true;
+SET hive.map.aggr=false;
+-- disabling map side aggregation as that can lead to different intermediate record counts
CREATE TABLE staging(t tinyint,
si smallint,
@@ -33,17 +35,16 @@ CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
-insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s;
+insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s;
-- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
-- which makes it hard to test bloom filters
-insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1;
-insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1;
+insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1;
+insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1;
CREATE TABLE orc_ppd(t tinyint,
si smallint,
@@ -56,12 +57,11 @@ CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
-insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s;
+insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s;
describe formatted orc_ppd;
http://git-wip-us.apache.org/repos/asf/hive/blob/7cca0978/ql/src/test/queries/clientpositive/orc_llap_counters1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters1.q b/ql/src/test/queries/clientpositive/orc_llap_counters1.q
index 06d6c4f..1e63c4f 100644
--- a/ql/src/test/queries/clientpositive/orc_llap_counters1.q
+++ b/ql/src/test/queries/clientpositive/orc_llap_counters1.q
@@ -3,6 +3,8 @@ SET hive.optimize.index.filter=true;
SET hive.cbo.enable=false;
SET hive.vectorized.execution.enabled=true;
SET hive.llap.io.enabled=true;
+SET hive.map.aggr=false;
+-- disabling map side aggregation as that can lead to different intermediate record counts
CREATE TABLE staging(t tinyint,
si smallint,
@@ -32,17 +34,16 @@ CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
-insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s;
+insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s;
-- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
-- which makes it hard to test bloom filters
-insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1;
-insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1;
+insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1;
+insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1;
CREATE TABLE orc_ppd(t tinyint,
si smallint,
@@ -55,12 +56,11 @@ CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
-insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s;
+insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s;
describe formatted orc_ppd;
http://git-wip-us.apache.org/repos/asf/hive/blob/7cca0978/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
index e1c1dd8..bc3915a 100644
--- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
@@ -57,7 +57,6 @@ PREHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
@@ -75,18 +74,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_ppd_staging
-PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s
+PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s
+POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -102,13 +100,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [(staging)staging.FieldSchema(name:i
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [(staging)staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.si SIMPLE [(staging)staging.FieldSchema(name:si, type:smallint, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.t SIMPLE [(staging)staging.FieldSchema(name:t, type:tinyint, comment:null), ]
-POSTHOOK: Lineage: orc_ppd_staging.ts SIMPLE [(staging)staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [(staging)staging.FieldSchema(name:s, type:string, comment:null), ]
-PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1
+PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1
+POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -124,13 +121,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION []
-POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION []
-PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1
+PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1
+POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -146,7 +142,6 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION []
-POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION []
PREHOOK: query: CREATE TABLE orc_ppd(t tinyint,
si smallint,
@@ -159,7 +154,6 @@ PREHOOK: query: CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
@@ -177,18 +171,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_ppd
-PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s
+PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd_staging
PREHOOK: Output: default@orc_ppd
-POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s
+POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_ppd_staging
POSTHOOK: Output: default@orc_ppd
@@ -197,14 +190,13 @@ POSTHOOK: Lineage: orc_ppd.bin SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSch
POSTHOOK: Lineage: orc_ppd.bo SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:bo, type:boolean, comment:null), ]
POSTHOOK: Lineage: orc_ppd.c EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd.d SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:d, type:double, comment:null), ]
-POSTHOOK: Lineage: orc_ppd.da EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_ppd.da SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:da, type:date, comment:null), ]
POSTHOOK: Lineage: orc_ppd.dec SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:dec, type:decimal(4,2), comment:null), ]
POSTHOOK: Lineage: orc_ppd.f SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:f, type:float, comment:null), ]
POSTHOOK: Lineage: orc_ppd.i SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:i, type:int, comment:null), ]
POSTHOOK: Lineage: orc_ppd.s SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd.si SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:si, type:smallint, comment:null), ]
POSTHOOK: Lineage: orc_ppd.t SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:t, type:tinyint, comment:null), ]
-POSTHOOK: Lineage: orc_ppd.ts SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
POSTHOOK: Lineage: orc_ppd.v EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
PREHOOK: query: describe formatted orc_ppd
PREHOOK: type: DESCTABLE
@@ -225,7 +217,6 @@ s string
c char(50)
v varchar(50)
da date
-ts timestamp
dec decimal(4,2)
bin binary
@@ -241,8 +232,8 @@ Table Parameters:
numRows 2100
orc.bloom.filter.columns *
orc.row.index.stride 1000
- rawDataSize 1223514
- totalSize 60010
+ rawDataSize 1139514
+ totalSize 57100
#### A masked pattern was here ####
# Storage Information
@@ -260,7 +251,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 16711
+ HDFS_BYTES_READ: 16677
HDFS_BYTES_WRITTEN: 104
HDFS_READ_OPS: 5
HDFS_LARGE_READ_OPS: 0
@@ -270,7 +261,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2100
Stage-1 LLAP IO COUNTERS:
METADATA_CACHE_MISS: 2
NUM_DECODED_BATCHES: 1
@@ -297,7 +288,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 1501
+ HDFS_BYTES_READ: 1467
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
@@ -307,7 +298,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 8
Stage-1 LLAP IO COUNTERS:
ALLOCATED_BYTES: 524288
ALLOCATED_USED_BYTES: 269
@@ -334,7 +325,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 22
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -359,7 +350,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 16
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -384,7 +375,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 18
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -434,7 +425,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 32
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -459,7 +450,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 32
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -484,7 +475,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 1697
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -509,7 +500,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 12
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -534,7 +525,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 1713
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -559,7 +550,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 6
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -584,7 +575,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 50
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -609,7 +600,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 318
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0
@@ -682,7 +673,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 0
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 0
Stage-1 LLAP IO COUNTERS:
METADATA_CACHE_HIT: 2
SELECTED_ROWGROUPS: 0
@@ -706,7 +697,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 6166
+ HDFS_BYTES_READ: 6132
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
@@ -716,7 +707,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
ALLOCATED_BYTES: 786432
ALLOCATED_USED_BYTES: 11299
@@ -744,7 +735,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 6
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -769,7 +760,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 6
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -808,7 +799,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2100
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -853,7 +844,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -878,7 +869,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -903,7 +894,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -928,7 +919,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 81
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -953,7 +944,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 74
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -978,7 +969,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 12
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -1003,7 +994,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 13
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -1053,7 +1044,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 7
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -1138,7 +1129,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -1163,7 +1154,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 6
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 3980
CACHE_MISS_BYTES: 0
@@ -1188,7 +1179,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 1000
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 4229
CACHE_MISS_BYTES: 0
@@ -1213,7 +1204,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 4229
CACHE_MISS_BYTES: 0
http://git-wip-us.apache.org/repos/asf/hive/blob/7cca0978/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out
index de686b0..5c54622 100644
--- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out
@@ -57,7 +57,6 @@ PREHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
@@ -75,18 +74,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_ppd_staging
-PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s
+PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s
+POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -102,13 +100,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [(staging)staging.FieldSchema(name:i
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [(staging)staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.si SIMPLE [(staging)staging.FieldSchema(name:si, type:smallint, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.t SIMPLE [(staging)staging.FieldSchema(name:t, type:tinyint, comment:null), ]
-POSTHOOK: Lineage: orc_ppd_staging.ts SIMPLE [(staging)staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [(staging)staging.FieldSchema(name:s, type:string, comment:null), ]
-PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1
+PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1
+POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -124,13 +121,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION []
-POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION []
-PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1
+PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@staging
PREHOOK: Output: default@orc_ppd_staging
-POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1
+POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@staging
POSTHOOK: Output: default@orc_ppd_staging
@@ -146,7 +142,6 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE []
POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION []
-POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION []
POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION []
PREHOOK: query: CREATE TABLE orc_ppd(t tinyint,
si smallint,
@@ -159,7 +154,6 @@ PREHOOK: query: CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
@@ -177,18 +171,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd(t tinyint,
c char(50),
v varchar(50),
da date,
- ts timestamp,
dec decimal(4,2),
bin binary)
STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@orc_ppd
-PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s
+PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s
PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd_staging
PREHOOK: Output: default@orc_ppd
-POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s
+POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_ppd_staging
POSTHOOK: Output: default@orc_ppd
@@ -197,14 +190,13 @@ POSTHOOK: Lineage: orc_ppd.bin SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSch
POSTHOOK: Lineage: orc_ppd.bo SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:bo, type:boolean, comment:null), ]
POSTHOOK: Lineage: orc_ppd.c EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd.d SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:d, type:double, comment:null), ]
-POSTHOOK: Lineage: orc_ppd.da EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_ppd.da SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:da, type:date, comment:null), ]
POSTHOOK: Lineage: orc_ppd.dec SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:dec, type:decimal(4,2), comment:null), ]
POSTHOOK: Lineage: orc_ppd.f SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:f, type:float, comment:null), ]
POSTHOOK: Lineage: orc_ppd.i SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:i, type:int, comment:null), ]
POSTHOOK: Lineage: orc_ppd.s SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
POSTHOOK: Lineage: orc_ppd.si SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:si, type:smallint, comment:null), ]
POSTHOOK: Lineage: orc_ppd.t SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:t, type:tinyint, comment:null), ]
-POSTHOOK: Lineage: orc_ppd.ts SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ]
POSTHOOK: Lineage: orc_ppd.v EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ]
PREHOOK: query: describe formatted orc_ppd
PREHOOK: type: DESCTABLE
@@ -225,7 +217,6 @@ s string
c char(50)
v varchar(50)
da date
-ts timestamp
dec decimal(4,2)
bin binary
@@ -241,8 +232,8 @@ Table Parameters:
numRows 2100
orc.bloom.filter.columns *
orc.row.index.stride 1000
- rawDataSize 1223514
- totalSize 60010
+ rawDataSize 1139514
+ totalSize 57100
#### A masked pattern was here ####
# Storage Information
@@ -260,7 +251,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 17909
+ HDFS_BYTES_READ: 17875
HDFS_BYTES_WRITTEN: 104
HDFS_READ_OPS: 6
HDFS_LARGE_READ_OPS: 0
@@ -270,7 +261,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2094
Stage-1 LLAP IO COUNTERS:
ALLOCATED_BYTES: 524288
ALLOCATED_USED_BYTES: 269
@@ -296,7 +287,7 @@ Stage-1 HIVE COUNTERS:
DESERIALIZE_ERRORS: 0
RECORDS_IN_Map_1: 2100
RECORDS_OUT_0: 1
- RECORDS_OUT_INTERMEDIATE_Map_1: 1
+ RECORDS_OUT_INTERMEDIATE_Map_1: 2094
Stage-1 LLAP IO COUNTERS:
CACHE_HIT_BYTES: 249
CACHE_MISS_BYTES: 0