You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/06 02:50:34 UTC
[28/53] [abbrv] hive git commit: HIVE-11185: Fix
compustat_avro.q/load_dyn_part14_win.q for Windows (Jason Dere,
reviewed by Hari Subramaniyan)
HIVE-11185: Fix compustat_avro.q/load_dyn_part14_win.q for Windows (Jason Dere, reviewed by Hari Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/607f23d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/607f23d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/607f23d9
Branch: refs/heads/llap
Commit: 607f23d90c8f94d13a64fae1e4f23e58dd3fb3a2
Parents: d0e868f
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Aug 3 13:43:57 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Aug 3 13:43:57 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/compustat_avro.q | 8 +-
.../clientpositive/load_dyn_part14_win.q | 18 +-
.../results/clientpositive/compustat_avro.q.out | 8 +-
.../clientpositive/load_dyn_part14_win.q.out | 167 ++++++++-----------
4 files changed, 95 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/queries/clientpositive/compustat_avro.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/compustat_avro.q b/ql/src/test/queries/clientpositive/compustat_avro.q
index 4d1b604..4d781a3 100644
--- a/ql/src/test/queries/clientpositive/compustat_avro.q
+++ b/ql/src/test/queries/clientpositive/compustat_avro.q
@@ -1,4 +1,10 @@
drop table if exists testAvro;
+
+dfs -cp ${system:hive.root}data/files/grad.avsc ${system:test.tmp.dir}/;
+
+-- File URIs using system:hive.root (using file:/) don't seem to work properly in DDL statements on Windows,
+-- so use dfs to copy them over to system:test.tmp.dir (which uses pfile:/), which does appear to work
+
create table testAvro
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
@@ -6,7 +12,7 @@ create table testAvro
'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
- TBLPROPERTIES ('avro.schema.url'='file://${system:hive.root}data/files/grad.avsc');
+ TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad.avsc');
describe formatted testAvro.col1;
http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/queries/clientpositive/load_dyn_part14_win.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/load_dyn_part14_win.q b/ql/src/test/queries/clientpositive/load_dyn_part14_win.q
old mode 100644
new mode 100755
index d32b539..8ff4772
--- a/ql/src/test/queries/clientpositive/load_dyn_part14_win.q
+++ b/ql/src/test/queries/clientpositive/load_dyn_part14_win.q
@@ -1,8 +1,9 @@
-- INCLUDE_OS_WINDOWS
-- included only on windows because of difference in file name encoding logic
+-- SORT_QUERY_RESULTS
-create table if not exists nzhang_part14 (key string)
+create table if not exists nzhang_part14 (key string)
partitioned by (value string);
describe extended nzhang_part14;
@@ -13,26 +14,25 @@ set hive.exec.dynamic.partition.mode=nonstrict;
explain
insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T;
insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T;
show partitions nzhang_part14;
-select * from nzhang_part14 where value <> 'a'
-order by key, value;
+select * from nzhang_part14 where value <> 'a';
http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/results/clientpositive/compustat_avro.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/compustat_avro.q.out b/ql/src/test/results/clientpositive/compustat_avro.q.out
index 8422108..f4becf1 100644
--- a/ql/src/test/results/clientpositive/compustat_avro.q.out
+++ b/ql/src/test/results/clientpositive/compustat_avro.q.out
@@ -2,7 +2,9 @@ PREHOOK: query: drop table if exists testAvro
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table if exists testAvro
POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table testAvro
+#### A masked pattern was here ####
+
+create table testAvro
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
STORED AS INPUTFORMAT
@@ -13,7 +15,9 @@ PREHOOK: query: create table testAvro
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@testAvro
-POSTHOOK: query: create table testAvro
+#### A masked pattern was here ####
+
+create table testAvro
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
STORED AS INPUTFORMAT
http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out b/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out
index e499552..d793e39 100644
--- a/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out
+++ b/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out
@@ -1,8 +1,9 @@
PREHOOK: query: -- INCLUDE_OS_WINDOWS
-- included only on windows because of difference in file name encoding logic
+-- SORT_QUERY_RESULTS
-create table if not exists nzhang_part14 (key string)
+create table if not exists nzhang_part14 (key string)
partitioned by (value string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -10,8 +11,9 @@ PREHOOK: Output: default@nzhang_part14
POSTHOOK: query: -- INCLUDE_OS_WINDOWS
-- included only on windows because of difference in file name encoding logic
+-- SORT_QUERY_RESULTS
-create table if not exists nzhang_part14 (key string)
+create table if not exists nzhang_part14 (key string)
partitioned by (value string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
@@ -34,21 +36,21 @@ value string
PREHOOK: query: explain
insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T
PREHOOK: type: QUERY
POSTHOOK: query: explain
insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -72,24 +74,21 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 'k2' (type: string), '' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 2
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 'k1' (type: string), null (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -102,49 +101,37 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.nzhang_part14
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.nzhang_part14
TableScan
Union
- Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.nzhang_part14
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.nzhang_part14
TableScan
Union
- Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.nzhang_part14
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.nzhang_part14
Stage: Stage-8
Conditional Operator
@@ -207,24 +194,21 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 'k3' (type: string), ' ' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 85500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 2
- Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 'k2' (type: string), '' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -239,24 +223,21 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 'k1' (type: string), UDFToString(null) (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 2
- Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 'k3' (type: string), ' ' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -266,22 +247,22 @@ STAGE PLANS:
PREHOOK: query: insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@nzhang_part14
POSTHOOK: query: insert overwrite table nzhang_part14 partition(value)
select key, value from (
- select 'k1' as key, cast(null as string) as value from src limit 2
+ select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a
union all
- select 'k2' as key, '' as value from src limit 2
+ select * from (select 'k2' as key, '' as value from src limit 2)b
union all
- select 'k3' as key, ' ' as value from src limit 2
+ select * from (select 'k3' as key, ' ' as value from src limit 2)c
) T
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -298,14 +279,12 @@ POSTHOOK: Input: default@nzhang_part14
value=%2520
value=__HIVE_DEFAULT_PARTITION__
PREHOOK: query: select * from nzhang_part14 where value <> 'a'
-order by key, value
PREHOOK: type: QUERY
PREHOOK: Input: default@nzhang_part14
PREHOOK: Input: default@nzhang_part14@value=%2520
PREHOOK: Input: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
POSTHOOK: query: select * from nzhang_part14 where value <> 'a'
-order by key, value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@nzhang_part14
POSTHOOK: Input: default@nzhang_part14@value=%2520