You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/12/07 20:17:16 UTC
[2/2] hive git commit: HIVE-15361: INSERT dynamic partition on S3
fails with a MoveTask failure (Sergio Pena,
reviewed by Mohit Sabharwal and Illya Yalovvy)
HIVE-15361: INSERT dynamic partition on S3 fails with a MoveTask failure (Sergio Pena, reviewed by Mohit Sabharwal and Illya Yalovvy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0978844
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0978844
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0978844
Branch: refs/heads/master
Commit: c0978844bc01c2eca52b8ae61783e9fd921b8b92
Parents: d60802d
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed Dec 7 11:06:52 2016 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Dec 7 14:16:38 2016 -0600
----------------------------------------------------------------------
.../insert_into_dynamic_partitions.q | 24 ++
.../queries/clientpositive/insert_into_table.q | 8 -
.../insert_overwrite_dynamic_partitions.q | 30 ++
.../clientpositive/insert_overwrite_table.q | 9 -
.../insert_into_dynamic_partitions.q.out | 333 +++++++++++++++
.../clientpositive/insert_into_table.q.out | 242 +----------
.../insert_overwrite_directory.q.out | 12 +-
.../insert_overwrite_dynamic_partitions.q.out | 413 +++++++++++++++++++
.../clientpositive/insert_overwrite_table.q.out | 263 +-----------
.../hive/ql/optimizer/GenMapRedUtils.java | 234 +++++------
.../apache/hadoop/hive/ql/plan/LoadDesc.java | 5 +-
.../hadoop/hive/ql/plan/LoadFileDesc.java | 10 +
.../hadoop/hive/ql/plan/LoadTableDesc.java | 12 +
.../apache/hadoop/hive/ql/plan/MoveWork.java | 10 +
...TestGenMapRedUtilsCreateConditionalTask.java | 108 ++++-
15 files changed, 1057 insertions(+), 656 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
new file mode 100644
index 0000000..7269e29
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
@@ -0,0 +1,24 @@
+SET hive.blobstore.optimizations.enabled=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+-- Single partition with buckets
+DROP TABLE table1;
+CREATE TABLE table1 (id int) partitioned by (key string) clustered by (id) into 2 buckets LOCATION '${hiveconf:test.blobstore.path.unique}/table1/';
+INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+SELECT * FROM table1;
+EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+DROP TABLE table1;
+
+-- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string);
+INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22);
+INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37);
+INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50);
+CREATE TABLE table2 (name string, age int) PARTITIONED BY (country string, state string) LOCATION '${hiveconf:test.blobstore.path.unique}/table2/';
+INSERT INTO TABLE table2 PARTITION (country, state) SELECT * FROM table1;
+INSERT INTO TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON');
+SHOW PARTITIONS table2;
+SELECT * FROM table2;
+DROP TABLE table2;
+DROP TABLE table1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_table.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_table.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_table.q
index 25e2e70..fd38aef 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_table.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_table.q
@@ -9,12 +9,4 @@ INSERT INTO TABLE table1 VALUES (1);
INSERT INTO TABLE table1 VALUES (2);
SELECT * FROM table1;
EXPLAIN EXTENDED INSERT INTO TABLE table1 VALUES (1);
-DROP TABLE table1;
-
--- Insert dynamic partitions;
-CREATE TABLE table1 (id int) partitioned by (key string) clustered by (id) into 2 buckets LOCATION '${hiveconf:test.blobstore.path.unique}/table1/';
-INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
-INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
-SELECT * FROM table1;
-EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
DROP TABLE table1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
new file mode 100644
index 0000000..0be4f16
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
@@ -0,0 +1,30 @@
+SET hive.blobstore.optimizations.enabled=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+-- Single partition with buckets
+DROP TABLE table1;
+CREATE TABLE table1 (id int) partitioned by (key string) clustered by (id) into 2 buckets LOCATION '${hiveconf:test.blobstore.path.unique}/table1/';
+INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+SELECT * FROM table1;
+INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+SELECT * FROM table1;
+EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
+DROP TABLE table1;
+
+-- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string);
+INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22);
+INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37);
+INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50);
+CREATE TABLE table2 (name string, age int) PARTITIONED BY (country string, state string) LOCATION '${hiveconf:test.blobstore.path.unique}/table2/';
+INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1;
+SHOW PARTITIONS table2;
+SELECT * FROM table2;
+INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1 WHERE age < 30;
+SHOW PARTITIONS table2;
+SELECT * FROM table2;
+INSERT OVERWRITE TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON');
+SHOW PARTITIONS table2;
+SELECT * FROM table2;
+DROP TABLE table2;
+DROP TABLE table1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_table.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_table.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_table.q
index 846b2b1..b6c289e 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_table.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_table.q
@@ -10,13 +10,4 @@ SELECT * FROM table1;
INSERT OVERWRITE TABLE table1 VALUES (2);
SELECT * FROM table1;
EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 VALUES (1);
-DROP TABLE table1;
-
--- Insert dynamic partitions;
-CREATE TABLE table1 (id int) partitioned by (key string) clustered by (id) into 2 buckets LOCATION '${hiveconf:test.blobstore.path.unique}/table1/';
-INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
-SELECT * FROM table1;
-INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
-SELECT * FROM table1;
-EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505');
DROP TABLE table1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
new file mode 100644
index 0000000..d7613f3
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -0,0 +1,333 @@
+PREHOOK: query: -- Single partition with buckets
+DROP TABLE table1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- Single partition with buckets
+DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/table1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/table1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1
+POSTHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@key=101
+POSTHOOK: Output: default@table1@key=202
+POSTHOOK: Output: default@table1@key=303
+POSTHOOK: Output: default@table1@key=404
+POSTHOOK: Output: default@table1@key=505
+POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1
+POSTHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@key=101
+POSTHOOK: Output: default@table1@key=202
+POSTHOOK: Output: default@table1@key=303
+POSTHOOK: Output: default@table1@key=404
+POSTHOOK: Output: default@table1@key=505
+POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SELECT * FROM table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@key=101
+PREHOOK: Input: default@table1@key=202
+PREHOOK: Input: default@table1@key=303
+PREHOOK: Input: default@table1@key=404
+PREHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@key=101
+POSTHOOK: Input: default@table1@key=202
+POSTHOOK: Input: default@table1@key=303
+POSTHOOK: Input: default@table1@key=404
+POSTHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+1 101
+1 101
+2 202
+2 202
+3 303
+3 303
+4 404
+4 404
+5 505
+5 505
+PREHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: values__tmp__table__3
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string), '_bucket_number' (type: string)
+ null sort order: aa
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: int)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: Values__Tmp__Table__3
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1,tmp_values_col2
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.values__tmp__table__3
+ serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1,tmp_values_col2
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.values__tmp__table__3
+ serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.values__tmp__table__3
+ name: default.values__tmp__table__3
+ Truncated Path -> Alias:
+#### A masked pattern was here ####
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, '_bucket_number'
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: ### BLOBSTORE_STAGING_PATH ###
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count 2
+ bucket_field_name id
+ columns id
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/table1
+ name default.table1
+ partition_columns key
+ partition_columns.types string
+ serialization.ddl struct table1 { i32 id}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.table1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ key
+ replace: false
+ source: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count 2
+ bucket_field_name id
+ columns id
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/table1
+ name default.table1
+ partition_columns key
+ partition_columns.types string
+ serialization.ddl struct table1 { i32 id}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.table1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+
+PREHOOK: query: DROP TABLE table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table1
+PREHOOK: Output: default@table1
+POSTHOOK: query: DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table1
+POSTHOOK: Output: default@table1
+PREHOOK: query: -- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+POSTHOOK: query: -- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).age EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).name SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).age EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).name SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=TX
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=TX
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=TX).age EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=TX).name SIMPLE [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/table2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table2
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/table2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table2
+PREHOOK: query: INSERT INTO TABLE table2 PARTITION (country, state) SELECT * FROM table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@country=USA/state=CA
+PREHOOK: Input: default@table1@country=USA/state=TX
+PREHOOK: Output: default@table2
+POSTHOOK: query: INSERT INTO TABLE table2 PARTITION (country, state) SELECT * FROM table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@country=USA/state=CA
+POSTHOOK: Input: default@table1@country=USA/state=TX
+POSTHOOK: Output: default@table2@country=USA/state=CA
+POSTHOOK: Output: default@table2@country=USA/state=TX
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).age SIMPLE [(table1)table1.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).name SIMPLE [(table1)table1.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=TX).age SIMPLE [(table1)table1.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=TX).name SIMPLE [(table1)table1.FieldSchema(name:name, type:string, comment:null), ]
+PREHOOK: query: INSERT INTO TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table2@country=MEX
+POSTHOOK: query: INSERT INTO TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table2@country=MEX/state=SON
+POSTHOOK: Lineage: table2 PARTITION(country=MEX,state=SON).age EXPRESSION [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table2 PARTITION(country=MEX,state=SON).name SIMPLE [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SHOW PARTITIONS table2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@table2
+POSTHOOK: query: SHOW PARTITIONS table2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@table2
+country=MEX/state=SON
+country=USA/state=CA
+country=USA/state=TX
+PREHOOK: query: SELECT * FROM table2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table2@country=MEX/state=SON
+PREHOOK: Input: default@table2@country=USA/state=CA
+PREHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table2@country=MEX/state=SON
+POSTHOOK: Input: default@table2@country=USA/state=CA
+POSTHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+Peter Mo 87 MEX SON
+Mirna Cage 37 USA CA
+Mark Cage 38 USA CA
+Jane Doe 22 USA CA
+John Doe 23 USA CA
+Maria Full 50 USA TX
+Bill Rose 52 USA TX
+PREHOOK: query: DROP TABLE table2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table2
+PREHOOK: Output: default@table2
+POSTHOOK: query: DROP TABLE table2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table2
+POSTHOOK: Output: default@table2
+PREHOOK: query: DROP TABLE table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table1
+PREHOOK: Output: default@table1
+POSTHOOK: query: DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table1
+POSTHOOK: Output: default@table1
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
index 223cdf4..fe7fdb0 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
@@ -44,9 +44,9 @@ STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
- Stage-2 depends on stages: Stage-0, Stage-4, Stage-6
+ Stage-2 depends on stages: Stage-0, Stage-4
Stage-3
- Stage-0 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3, Stage-6
Stage-5
Stage-6 depends on stages: Stage-5
@@ -344,242 +344,10 @@ STAGE PLANS:
Stage: Stage-6
Move Operator
- tables:
- replace: false
- source: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- numFiles 2
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 4
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
-
-PREHOOK: query: DROP TABLE table1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@table1
-PREHOOK: Output: default@table1
-POSTHOOK: query: DROP TABLE table1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@table1
-POSTHOOK: Output: default@table1
-PREHOOK: query: -- Insert dynamic partitions;
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/table1
-PREHOOK: Output: database:default
-PREHOOK: Output: default@table1
-POSTHOOK: query: -- Insert dynamic partitions;
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/table1
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@table1
-PREHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@table1
-POSTHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@table1@key=101
-POSTHOOK: Output: default@table1@key=202
-POSTHOOK: Output: default@table1@key=303
-POSTHOOK: Output: default@table1@key=404
-POSTHOOK: Output: default@table1@key=505
-POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@table1
-POSTHOOK: query: INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@table1@key=101
-POSTHOOK: Output: default@table1@key=202
-POSTHOOK: Output: default@table1@key=303
-POSTHOOK: Output: default@table1@key=404
-POSTHOOK: Output: default@table1@key=505
-POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: SELECT * FROM table1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@table1
-PREHOOK: Input: default@table1@key=101
-PREHOOK: Input: default@table1@key=202
-PREHOOK: Input: default@table1@key=303
-PREHOOK: Input: default@table1@key=404
-PREHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM table1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@table1
-POSTHOOK: Input: default@table1@key=101
-POSTHOOK: Input: default@table1@key=202
-POSTHOOK: Input: default@table1@key=303
-POSTHOOK: Input: default@table1@key=404
-POSTHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-1 101
-1 101
-2 202
-2 202
-3 303
-3 303
-4 404
-4 404
-5 505
-5 505
-PREHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: values__tmp__table__6
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Select Operator
- expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: string), '_bucket_number' (type: string)
- null sort order: aa
- sort order: ++
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col0 (type: int)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: Values__Tmp__Table__6
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns tmp_values_col1,tmp_values_col2
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.values__tmp__table__6
- serialization.ddl struct values__tmp__table__6 { string tmp_values_col1, string tmp_values_col2}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns tmp_values_col1,tmp_values_col2
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.values__tmp__table__6
- serialization.ddl struct values__tmp__table__6 { string tmp_values_col1, string tmp_values_col2}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.values__tmp__table__6
- name: default.values__tmp__table__6
- Truncated Path -> Alias:
-#### A masked pattern was here ####
- Needs Tagging: false
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY.'_bucket_number' (type: string)
- outputColumnNames: _col0, _col1, '_bucket_number'
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 1
- directory: ### BLOBSTORE_STAGING_PATH ###
- NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count 2
- bucket_field_name id
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- partition_columns key
- partition_columns.types string
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
- TotalFiles: 1
- GatherStats: true
- MultiFileSpray: false
-
- Stage: Stage-0
- Move Operator
- tables:
- partition:
- key
- replace: false
+ files:
+ hdfs directory: true
source: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count 2
- bucket_field_name id
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- partition_columns key
- partition_columns.types string
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
-
- Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ destination: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: DROP TABLE table1
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
index 9b993a6..82fb95d 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
@@ -71,7 +71,7 @@ STAGE DEPENDENCIES:
Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4
Stage-3
Stage-2
- Stage-0 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-2, Stage-5
Stage-4
Stage-5 depends on stages: Stage-4
@@ -276,7 +276,7 @@ STAGE PLANS:
files:
hdfs directory: true
source: ### BLOBSTORE_STAGING_PATH ###
- destination: ### test.blobstore.path ###/table1.dir
+ destination: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: EXPLAIN EXTENDED FROM table1
INSERT OVERWRITE DIRECTORY '### test.blobstore.path ###/table1.dir/' SELECT id
@@ -291,13 +291,13 @@ STAGE DEPENDENCIES:
Stage-7 depends on stages: Stage-2 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-3
- Stage-0 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3, Stage-6
Stage-5
Stage-6 depends on stages: Stage-5
Stage-12 depends on stages: Stage-2 , consists of Stage-9, Stage-8, Stage-10
Stage-9
Stage-8
- Stage-1 depends on stages: Stage-8
+ Stage-1 depends on stages: Stage-8, Stage-11
Stage-10
Stage-11 depends on stages: Stage-10
@@ -525,7 +525,7 @@ STAGE PLANS:
files:
hdfs directory: true
source: ### BLOBSTORE_STAGING_PATH ###
- destination: ### test.blobstore.path ###/table1.dir
+ destination: ### BLOBSTORE_STAGING_PATH ###
Stage: Stage-12
Conditional Operator
@@ -645,5 +645,5 @@ STAGE PLANS:
files:
hdfs directory: true
source: ### BLOBSTORE_STAGING_PATH ###
- destination: ### test.blobstore.path ###/table2.dir
+ destination: ### BLOBSTORE_STAGING_PATH ###
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
new file mode 100644
index 0000000..4d0c153
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -0,0 +1,413 @@
+PREHOOK: query: -- Single partition with buckets
+DROP TABLE table1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- Single partition with buckets
+DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/table1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/table1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1
+POSTHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@key=101
+POSTHOOK: Output: default@table1@key=202
+POSTHOOK: Output: default@table1@key=303
+POSTHOOK: Output: default@table1@key=404
+POSTHOOK: Output: default@table1@key=505
+POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SELECT * FROM table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@key=101
+PREHOOK: Input: default@table1@key=202
+PREHOOK: Input: default@table1@key=303
+PREHOOK: Input: default@table1@key=404
+PREHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@key=101
+POSTHOOK: Input: default@table1@key=202
+POSTHOOK: Input: default@table1@key=303
+POSTHOOK: Input: default@table1@key=404
+POSTHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+1 101
+2 202
+3 303
+4 404
+5 505
+PREHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1
+POSTHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@key=101
+POSTHOOK: Output: default@table1@key=202
+POSTHOOK: Output: default@table1@key=303
+POSTHOOK: Output: default@table1@key=404
+POSTHOOK: Output: default@table1@key=505
+POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SELECT * FROM table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@key=101
+PREHOOK: Input: default@table1@key=202
+PREHOOK: Input: default@table1@key=303
+PREHOOK: Input: default@table1@key=404
+PREHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@key=101
+POSTHOOK: Input: default@table1@key=202
+POSTHOOK: Input: default@table1@key=303
+POSTHOOK: Input: default@table1@key=404
+POSTHOOK: Input: default@table1@key=505
+#### A masked pattern was here ####
+1 101
+2 202
+3 303
+4 404
+5 505
+PREHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: values__tmp__table__3
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string), '_bucket_number' (type: string)
+ null sort order: aa
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: int)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: Values__Tmp__Table__3
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1,tmp_values_col2
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.values__tmp__table__3
+ serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1,tmp_values_col2
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.values__tmp__table__3
+ serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.values__tmp__table__3
+ name: default.values__tmp__table__3
+ Truncated Path -> Alias:
+#### A masked pattern was here ####
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, '_bucket_number'
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: ### BLOBSTORE_STAGING_PATH ###
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count 2
+ bucket_field_name id
+ columns id
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/table1
+ name default.table1
+ partition_columns key
+ partition_columns.types string
+ serialization.ddl struct table1 { i32 id}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.table1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ key
+ replace: true
+ source: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count 2
+ bucket_field_name id
+ columns id
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/table1
+ name default.table1
+ partition_columns key
+ partition_columns.types string
+ serialization.ddl struct table1 { i32 id}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.table1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+
+PREHOOK: query: DROP TABLE table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table1
+PREHOOK: Output: default@table1
+POSTHOOK: query: DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table1
+POSTHOOK: Output: default@table1
+PREHOOK: query: -- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+POSTHOOK: query: -- Multiple partitions
+CREATE TABLE table1 (name string, age int) PARTITIONED BY (country string, state string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('John Doe', 23), ('Jane Doe', 22)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).age EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).name SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='CA') values ('Mark Cage', 38), ('Mirna Cage', 37)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=CA
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).age EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=CA).name SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1@country=USA/state=TX
+POSTHOOK: query: INSERT INTO table1 PARTITION (country='USA', state='TX') values ('Bill Rose', 52), ('Maria Full', 50)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1@country=USA/state=TX
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=TX).age EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1 PARTITION(country=USA,state=TX).name SIMPLE [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/table2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table2
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/table2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table2
+PREHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@country=USA/state=CA
+PREHOOK: Input: default@table1@country=USA/state=TX
+PREHOOK: Output: default@table2
+POSTHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@country=USA/state=CA
+POSTHOOK: Input: default@table1@country=USA/state=TX
+POSTHOOK: Output: default@table2@country=USA/state=CA
+POSTHOOK: Output: default@table2@country=USA/state=TX
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).age SIMPLE [(table1)table1.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).name SIMPLE [(table1)table1.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=TX).age SIMPLE [(table1)table1.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=TX).name SIMPLE [(table1)table1.FieldSchema(name:name, type:string, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS table2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@table2
+POSTHOOK: query: SHOW PARTITIONS table2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@table2
+country=USA/state=CA
+country=USA/state=TX
+PREHOOK: query: SELECT * FROM table2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table2@country=USA/state=CA
+PREHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table2@country=USA/state=CA
+POSTHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+Mirna Cage 37 USA CA
+Mark Cage 38 USA CA
+Jane Doe 22 USA CA
+John Doe 23 USA CA
+Maria Full 50 USA TX
+Bill Rose 52 USA TX
+PREHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1 WHERE age < 30
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1@country=USA/state=CA
+PREHOOK: Input: default@table1@country=USA/state=TX
+PREHOOK: Output: default@table2
+POSTHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1 WHERE age < 30
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1@country=USA/state=CA
+POSTHOOK: Input: default@table1@country=USA/state=TX
+POSTHOOK: Output: default@table2@country=USA/state=CA
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).age SIMPLE [(table1)table1.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: table2 PARTITION(country=USA,state=CA).name SIMPLE [(table1)table1.FieldSchema(name:name, type:string, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS table2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@table2
+POSTHOOK: query: SHOW PARTITIONS table2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@table2
+country=USA/state=CA
+country=USA/state=TX
+PREHOOK: query: SELECT * FROM table2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table2@country=USA/state=CA
+PREHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table2@country=USA/state=CA
+POSTHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+Jane Doe 22 USA CA
+John Doe 23 USA CA
+Maria Full 50 USA TX
+Bill Rose 52 USA TX
+PREHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table2@country=MEX
+POSTHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table2@country=MEX/state=SON
+POSTHOOK: Lineage: table2 PARTITION(country=MEX,state=SON).age EXPRESSION [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table2 PARTITION(country=MEX,state=SON).name SIMPLE [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SHOW PARTITIONS table2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@table2
+POSTHOOK: query: SHOW PARTITIONS table2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@table2
+country=MEX/state=SON
+country=USA/state=CA
+country=USA/state=TX
+PREHOOK: query: SELECT * FROM table2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table2@country=MEX/state=SON
+PREHOOK: Input: default@table2@country=USA/state=CA
+PREHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM table2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table2@country=MEX/state=SON
+POSTHOOK: Input: default@table2@country=USA/state=CA
+POSTHOOK: Input: default@table2@country=USA/state=TX
+#### A masked pattern was here ####
+Peter Mo 87 MEX SON
+Jane Doe 22 USA CA
+John Doe 23 USA CA
+Maria Full 50 USA TX
+Bill Rose 52 USA TX
+PREHOOK: query: DROP TABLE table2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table2
+PREHOOK: Output: default@table2
+POSTHOOK: query: DROP TABLE table2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table2
+POSTHOOK: Output: default@table2
+PREHOOK: query: DROP TABLE table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table1
+PREHOOK: Output: default@table1
+POSTHOOK: query: DROP TABLE table1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table1
+POSTHOOK: Output: default@table1
http://git-wip-us.apache.org/repos/asf/hive/blob/c0978844/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
index 81bcc76..82ae25f 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
@@ -52,9 +52,9 @@ STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
- Stage-2 depends on stages: Stage-0, Stage-4, Stage-6
+ Stage-2 depends on stages: Stage-0, Stage-4
Stage-3
- Stage-0 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3, Stage-6
Stage-5
Stage-6 depends on stages: Stage-5
@@ -379,263 +379,10 @@ STAGE PLANS:
Stage: Stage-6
Move Operator
- tables:
- replace: true
+ files:
+ hdfs directory: true
source: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
- bucket_count -1
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- numFiles 1
- numRows 1
- rawDataSize 1
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 2
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
-
-PREHOOK: query: DROP TABLE table1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@table1
-PREHOOK: Output: default@table1
-POSTHOOK: query: DROP TABLE table1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@table1
-POSTHOOK: Output: default@table1
-PREHOOK: query: -- Insert dynamic partitions;
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/table1
-PREHOOK: Output: database:default
-PREHOOK: Output: default@table1
-POSTHOOK: query: -- Insert dynamic partitions;
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/table1
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@table1
-PREHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@table1
-POSTHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@table1@key=101
-POSTHOOK: Output: default@table1@key=202
-POSTHOOK: Output: default@table1@key=303
-POSTHOOK: Output: default@table1@key=404
-POSTHOOK: Output: default@table1@key=505
-POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: SELECT * FROM table1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@table1
-PREHOOK: Input: default@table1@key=101
-PREHOOK: Input: default@table1@key=202
-PREHOOK: Input: default@table1@key=303
-PREHOOK: Input: default@table1@key=404
-PREHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM table1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@table1
-POSTHOOK: Input: default@table1@key=101
-POSTHOOK: Input: default@table1@key=202
-POSTHOOK: Input: default@table1@key=303
-POSTHOOK: Input: default@table1@key=404
-POSTHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-1 101
-2 202
-3 303
-4 404
-5 505
-PREHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@table1
-POSTHOOK: query: INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@table1@key=101
-POSTHOOK: Output: default@table1@key=202
-POSTHOOK: Output: default@table1@key=303
-POSTHOOK: Output: default@table1@key=404
-POSTHOOK: Output: default@table1@key=505
-POSTHOOK: Lineage: table1 PARTITION(key=101).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=202).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=303).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=404).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: table1 PARTITION(key=505).id EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: SELECT * FROM table1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@table1
-PREHOOK: Input: default@table1@key=101
-PREHOOK: Input: default@table1@key=202
-PREHOOK: Input: default@table1@key=303
-PREHOOK: Input: default@table1@key=404
-PREHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM table1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@table1
-POSTHOOK: Input: default@table1@key=101
-POSTHOOK: Input: default@table1@key=202
-POSTHOOK: Input: default@table1@key=303
-POSTHOOK: Input: default@table1@key=404
-POSTHOOK: Input: default@table1@key=505
-#### A masked pattern was here ####
-1 101
-2 202
-3 303
-4 404
-5 505
-PREHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key) VALUES (1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: values__tmp__table__6
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Select Operator
- expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: string), '_bucket_number' (type: string)
- null sort order: aa
- sort order: ++
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col0 (type: int)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: Values__Tmp__Table__6
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns tmp_values_col1,tmp_values_col2
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.values__tmp__table__6
- serialization.ddl struct values__tmp__table__6 { string tmp_values_col1, string tmp_values_col2}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns tmp_values_col1,tmp_values_col2
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.values__tmp__table__6
- serialization.ddl struct values__tmp__table__6 { string tmp_values_col1, string tmp_values_col2}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.values__tmp__table__6
- name: default.values__tmp__table__6
- Truncated Path -> Alias:
-#### A masked pattern was here ####
- Needs Tagging: false
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY.'_bucket_number' (type: string)
- outputColumnNames: _col0, _col1, '_bucket_number'
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 1
- directory: ### BLOBSTORE_STAGING_PATH ###
- NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count 2
- bucket_field_name id
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- partition_columns key
- partition_columns.types string
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
- TotalFiles: 1
- GatherStats: true
- MultiFileSpray: false
-
- Stage: Stage-0
- Move Operator
- tables:
- partition:
- key
- replace: true
- source: ### BLOBSTORE_STAGING_PATH ###
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count 2
- bucket_field_name id
- columns id
- columns.comments
- columns.types int
-#### A masked pattern was here ####
- location ### test.blobstore.path ###/table1
- name default.table1
- partition_columns key
- partition_columns.types string
- serialization.ddl struct table1 { i32 id}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.table1
-
- Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ destination: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: DROP TABLE table1
PREHOOK: type: DROPTABLE