You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/04/20 18:08:17 UTC
hive git commit: HIVE-16454: Add blobstore tests for inserting empty
into dynamic partition/list bucket tables & inserting cross blobstore tables
(Rentao Wu, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master 8e2f1830a -> bde615234
HIVE-16454: Add blobstore tests for inserting empty into dynamic partition/list bucket tables & inserting cross blobstore tables (Rentao Wu, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bde61523
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bde61523
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bde61523
Branch: refs/heads/master
Commit: bde6152344f3993a341b4834e435ba44172c7dc4
Parents: 8e2f183
Author: Rentao Wu <re...@amazon.com>
Authored: Thu Apr 20 13:07:51 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Thu Apr 20 13:07:51 2017 -0500
----------------------------------------------------------------------
.../insert_blobstore_to_blobstore.q | 29 ++++
.../insert_empty_into_blobstore.q | 53 +++++++
.../insert_blobstore_to_blobstore.q.out | 110 +++++++++++++
.../insert_empty_into_blobstore.q.out | 155 +++++++++++++++++++
4 files changed, 347 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bde61523/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
new file mode 100644
index 0000000..8219ee2
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q
@@ -0,0 +1,29 @@
+-- Test inserting into a blobstore table from another blobstore table.
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source (
+ a string,
+ b string,
+ c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_source';
+
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE blobstore_table;
+CREATE TABLE blobstore_table LIKE blobstore_source
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_table';
+
+INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source;
+
+SELECT COUNT(*) FROM blobstore_table;
+
+-- INSERT INTO should append all records to existing ones.
+INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source;
+
+SELECT COUNT(*) FROM blobstore_table;
+
+SELECT * FROM blobstore_table;
http://git-wip-us.apache.org/repos/asf/hive/blob/bde61523/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
new file mode 100644
index 0000000..d4f0c71
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q
@@ -0,0 +1,53 @@
+-- Test inserting empty rows into dynamic partitioned and list bucketed blobstore tables
+
+SET hive.blobstore.optimizations.enabled=true;
+
+DROP TABLE empty;
+DROP TABLE blobstore_dynamic_partitioning;
+DROP TABLE blobstore_list_bucketing;
+
+CREATE TABLE empty (
+ id int,
+ name string,
+ dept string,
+ pt string,
+ dt string,
+ hr string);
+
+CREATE TABLE blobstore_dynamic_partitioning (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_dynamic_partitioning';
+
+INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty;
+
+SELECT COUNT(*) FROM blobstore_dynamic_partitioning;
+
+CREATE TABLE blobstore_list_bucketing (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing';
+
+INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty;
+
+SELECT COUNT(*) FROM blobstore_list_bucketing;
+
+-- Now test empty inserts with blobstore optimizations turned off. This should give us same results.
+SET hive.blobstore.optimizations.enabled=false;
+
+INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty;
+SELECT COUNT(*) FROM blobstore_dynamic_partitioning;
+
+INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty;
+SELECT COUNT(*) FROM blobstore_list_bucketing;
http://git-wip-us.apache.org/repos/asf/hive/blob/bde61523/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
new file mode 100644
index 0000000..4651899
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out
@@ -0,0 +1,110 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source (
+ a string,
+ b string,
+ c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source (
+ a string,
+ b string,
+ c double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE blobstore_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+7
+PREHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@blobstore_table
+POSTHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@blobstore_table
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+14
+PREHOOK: query: SELECT * FROM blobstore_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM blobstore_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_table
+#### A masked pattern was here ####
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
http://git-wip-us.apache.org/repos/asf/hive/blob/bde61523/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
new file mode 100644
index 0000000..8e5e096
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out
@@ -0,0 +1,155 @@
+PREHOOK: query: DROP TABLE empty
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE blobstore_dynamic_partitioning
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_dynamic_partitioning
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE blobstore_list_bucketing
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_list_bucketing
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE empty (
+ id int,
+ name string,
+ dept string,
+ pt string,
+ dt string,
+ hr string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@empty
+POSTHOOK: query: CREATE TABLE empty (
+ id int,
+ name string,
+ dept string,
+ pt string,
+ dt string,
+ hr string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@empty
+PREHOOK: query: CREATE TABLE blobstore_dynamic_partitioning (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_dynamic_partitioning
+POSTHOOK: query: CREATE TABLE blobstore_dynamic_partitioning (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_dynamic_partitioning
+PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=a
+POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+0
+PREHOOK: query: CREATE TABLE blobstore_list_bucketing (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_list_bucketing
+POSTHOOK: query: CREATE TABLE blobstore_list_bucketing (
+ id int,
+ name string,
+ dept string)
+PARTITIONED BY (
+ pt string,
+ dt string,
+ hr string)
+SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_list_bucketing
+PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a
+POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+POSTHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+0
+PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=b
+POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_dynamic_partitioning
+#### A masked pattern was here ####
+0
+PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty
+PREHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b
+POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty
+POSTHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).dept SIMPLE [(empty)empty.FieldSchema(name:dept, type:string, comment:null), ]
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).id SIMPLE [(empty)empty.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).name SIMPLE [(empty)empty.FieldSchema(name:name, type:string, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_list_bucketing
+#### A masked pattern was here ####
+0