You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/05 20:38:13 UTC
[1/7] hive git commit: HIVE-19938: Upgrade scripts for information
schema (Daniel Dai, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master-txnstats 5225cef59 -> f384d4cac
HIVE-19938: Upgrade scripts for information schema (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ee8c72ae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ee8c72ae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ee8c72ae
Branch: refs/heads/master-txnstats
Commit: ee8c72ae171a7b571d7d6429d0052dcf47b3ed5d
Parents: e972122
Author: Daniel Dai <da...@gmail.com>
Authored: Tue Jul 3 23:05:28 2018 -0700
Committer: Daniel Dai <da...@gmail.com>
Committed: Tue Jul 3 23:05:28 2018 -0700
----------------------------------------------------------------------
.../upgrade/hive/hive-schema-3.1.0.hive.sql | 32 +++-----
.../hive/upgrade-3.0.0-to-3.1.0.hive.sql | 10 +++
.../scripts/upgrade/hive/upgrade.order.hive | 1 +
packaging/src/main/assembly/bin.xml | 3 +-
.../clientpositive/llap/resourceplan.q.out | 84 ++++++++------------
.../llap/strict_managed_tables_sysdb.q.out | 84 ++++++++------------
.../hive/metastore/tools/HiveSchemaHelper.java | 4 +-
7 files changed, 90 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql
index 9bfe9c2..f98894f 100644
--- a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql
@@ -1,7 +1,6 @@
-- HIVE system db
-DROP DATABASE IF EXISTS SYS CASCADE;
-CREATE DATABASE SYS;
+CREATE DATABASE IF NOT EXISTS SYS;
USE SYS;
@@ -817,14 +816,8 @@ FROM
\"PART_COL_STATS\""
);
-CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` (
- `VER_ID` BIGINT,
- `SCHEMA_VERSION` string,
- `VERSION_COMMENT` string,
- CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE
-);
-
-INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0');
+CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.1.0' AS `VERSION_COMMENT`;
CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` (
`VER_ID` BIGINT,
@@ -924,7 +917,7 @@ FROM
\"KEY_CONSTRAINTS\""
);
-CREATE VIEW `TABLE_STATS_VIEW` AS
+CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS
SELECT
`TBL_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -935,7 +928,7 @@ SELECT
max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME
FROM `TABLE_PARAMS` GROUP BY `TBL_ID`;
-CREATE VIEW `PARTITION_STATS_VIEW` AS
+CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS
SELECT
`PART_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -1064,12 +1057,11 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I
"
);
-DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE;
-CREATE DATABASE INFORMATION_SCHEMA;
+CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA;
USE INFORMATION_SCHEMA;
-CREATE VIEW IF NOT EXISTS `SCHEMATA`
+CREATE OR REPLACE VIEW `SCHEMATA`
(
`CATALOG_NAME`,
`SCHEMA_NAME`,
@@ -1097,7 +1089,7 @@ WHERE
OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
AND current_authorizer() = P.`AUTHORIZER`;
-CREATE VIEW IF NOT EXISTS `TABLES`
+CREATE OR REPLACE VIEW `TABLES`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -1134,7 +1126,7 @@ WHERE
OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')))
AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer();
-CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
+CREATE OR REPLACE VIEW `TABLE_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -1168,7 +1160,7 @@ WHERE
OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')))
AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer();
-CREATE VIEW IF NOT EXISTS `COLUMNS`
+CREATE OR REPLACE VIEW `COLUMNS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -1323,7 +1315,7 @@ WHERE
OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer());
-CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
+CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -1359,7 +1351,7 @@ WHERE
OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))
AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer());
-CREATE VIEW IF NOT EXISTS `VIEWS`
+CREATE OR REPLACE VIEW `VIEWS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql
new file mode 100644
index 0000000..ac93a12
--- /dev/null
+++ b/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql
@@ -0,0 +1,10 @@
+SELECT 'Upgrading MetaStore schema from 3.0.0 to 3.1.0';
+
+USE SYS;
+
+DROP TABLE IF EXISTS `VERSION`;
+
+CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.0.0' AS `VERSION_COMMENT`;
+
+SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0';
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/metastore/scripts/upgrade/hive/upgrade.order.hive
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/hive/upgrade.order.hive b/metastore/scripts/upgrade/hive/upgrade.order.hive
index e69de29..dd040af 100644
--- a/metastore/scripts/upgrade/hive/upgrade.order.hive
+++ b/metastore/scripts/upgrade/hive/upgrade.order.hive
@@ -0,0 +1 @@
+3.0.0-to-3.1.0
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index a9557cf..2dd9260 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -215,7 +215,8 @@
<include>**/*</include>
</includes>
<excludes>
- <exclude>**/upgrade.order.*</exclude> <!-- pick up upgrade order from standalone metastore-->
+ <!-- pick up upgrade order from standalone metastore, except hive, which does not exist in standalone metastore -->
+ <exclude>%regex[(!hive)/upgrade.order.*]</exclude>
</excludes>
<outputDirectory>scripts/metastore/upgrade</outputDirectory>
</fileSet>
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/ql/src/test/results/clientpositive/llap/resourceplan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
index 3f59343..3933273 100644
--- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
+++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
@@ -66,14 +66,10 @@ default srcpart hive_test_user USER DELETE true -1 hive_test_user
default srcpart hive_test_user USER INSERT true -1 hive_test_user
default srcpart hive_test_user USER SELECT true -1 hive_test_user
default srcpart hive_test_user USER UPDATE true -1 hive_test_user
-PREHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE
-PREHOOK: type: DROPDATABASE
-POSTHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE
-POSTHOOK: type: DROPDATABASE
-PREHOOK: query: CREATE DATABASE SYS
+PREHOOK: query: CREATE DATABASE IF NOT EXISTS SYS
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:SYS
-POSTHOOK: query: CREATE DATABASE SYS
+POSTHOOK: query: CREATE DATABASE IF NOT EXISTS SYS
POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:SYS
PREHOOK: query: USE SYS
@@ -1842,35 +1838,21 @@ FROM
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@PART_COL_STATS
POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` (
- `VER_ID` BIGINT,
- `SCHEMA_VERSION` string,
- `VERSION_COMMENT` string,
- CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE
-)
-PREHOOK: type: CREATETABLE
+PREHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.1.0' AS `VERSION_COMMENT`
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: SYS@VERSION
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` (
- `VER_ID` BIGINT,
- `SCHEMA_VERSION` string,
- `VERSION_COMMENT` string,
- CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE
-)
-POSTHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.1.0' AS `VERSION_COMMENT`
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: SYS@VERSION
POSTHOOK: Output: database:sys
-PREHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: sys@version
-POSTHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: sys@version
-POSTHOOK: Lineage: version.schema_version SCRIPT []
-POSTHOOK: Lineage: version.ver_id SCRIPT []
-POSTHOOK: Lineage: version.version_comment SCRIPT []
+POSTHOOK: Lineage: VERSION.schema_version SIMPLE []
+POSTHOOK: Lineage: VERSION.ver_id SIMPLE []
+POSTHOOK: Lineage: VERSION.version_comment SIMPLE []
PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` (
`VER_ID` BIGINT,
`SCHEMA_VERSION` string,
@@ -2053,7 +2035,7 @@ FROM
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@KEY_CONSTRAINTS
POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS
SELECT
`TBL_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2067,7 +2049,7 @@ PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@table_params
PREHOOK: Output: SYS@TABLE_STATS_VIEW
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS
SELECT
`TBL_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2088,7 +2070,7 @@ POSTHOOK: Lineage: TABLE_STATS_VIEW.raw_data_size EXPRESSION [(table_params)tabl
POSTHOOK: Lineage: TABLE_STATS_VIEW.tbl_id SIMPLE [(table_params)table_params.FieldSchema(name:tbl_id, type:bigint, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_STATS_VIEW.total_size EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_STATS_VIEW.transient_last_ddl_time EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS
+PREHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS
SELECT
`PART_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2102,7 +2084,7 @@ PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@partition_params
PREHOOK: Output: SYS@PARTITION_STATS_VIEW
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS
+POSTHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS
SELECT
`PART_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2379,14 +2361,10 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_MAPPINGS
POSTHOOK: Output: database:sys
-PREHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE
-PREHOOK: type: DROPDATABASE
-POSTHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE
-POSTHOOK: type: DROPDATABASE
-PREHOOK: query: CREATE DATABASE INFORMATION_SCHEMA
+PREHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:INFORMATION_SCHEMA
-POSTHOOK: query: CREATE DATABASE INFORMATION_SCHEMA
+POSTHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA
POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:INFORMATION_SCHEMA
PREHOOK: query: USE INFORMATION_SCHEMA
@@ -2395,7 +2373,7 @@ PREHOOK: Input: database:information_schema
POSTHOOK: query: USE INFORMATION_SCHEMA
POSTHOOK: type: SWITCHDATABASE
POSTHOOK: Input: database:information_schema
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
+PREHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA`
(
`CATALOG_NAME`,
`SCHEMA_NAME`,
@@ -2428,7 +2406,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@SCHEMATA
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
+POSTHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA`
(
`CATALOG_NAME`,
`SCHEMA_NAME`,
@@ -2468,7 +2446,7 @@ POSTHOOK: Lineage: SCHEMATA.default_character_set_schema EXPRESSION []
POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
#### A masked pattern was here ####
POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)d.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLES`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2510,7 +2488,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@TABLES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLES`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2564,7 +2542,7 @@ POSTHOOK: Lineage: TABLES.table_type EXPRESSION [(tbls)t.FieldSchema(name:view_o
POSTHOOK: Lineage: TABLES.user_defined_type_catalog EXPRESSION []
POSTHOOK: Lineage: TABLES.user_defined_type_name EXPRESSION []
POSTHOOK: Lineage: TABLES.user_defined_type_schema EXPRESSION []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -2603,7 +2581,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@TABLE_PRIVILEGES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -2650,7 +2628,7 @@ POSTHOOK: Lineage: TABLE_PRIVILEGES.table_catalog SIMPLE []
POSTHOOK: Lineage: TABLE_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_PRIVILEGES.with_hierarchy SIMPLE []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
+PREHOOK: query: CREATE OR REPLACE VIEW `COLUMNS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2812,7 +2790,7 @@ PREHOOK: Input: sys@tbl_col_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMNS
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
+POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMNS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -3021,7 +2999,7 @@ POSTHOOK: Lineage: COLUMNS.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, ty
POSTHOOK: Lineage: COLUMNS.udt_catalog EXPRESSION []
POSTHOOK: Lineage: COLUMNS.udt_name EXPRESSION []
POSTHOOK: Lineage: COLUMNS.udt_schema EXPRESSION []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
+PREHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -3064,7 +3042,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -3115,7 +3093,7 @@ POSTHOOK: Lineage: COLUMN_PRIVILEGES.privilege_type SIMPLE [(tbl_col_privs)p.Fie
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_catalog SIMPLE []
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
+PREHOOK: query: CREATE OR REPLACE VIEW `VIEWS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -3157,7 +3135,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@VIEWS
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
+POSTHOOK: query: CREATE OR REPLACE VIEW `VIEWS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
index 52eaf42..7637f0e 100644
--- a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
@@ -156,14 +156,10 @@ default srcpart hive_test_user USER DELETE true -1 hive_test_user
default srcpart hive_test_user USER INSERT true -1 hive_test_user
default srcpart hive_test_user USER SELECT true -1 hive_test_user
default srcpart hive_test_user USER UPDATE true -1 hive_test_user
-PREHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE
-PREHOOK: type: DROPDATABASE
-POSTHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE
-POSTHOOK: type: DROPDATABASE
-PREHOOK: query: CREATE DATABASE SYS
+PREHOOK: query: CREATE DATABASE IF NOT EXISTS SYS
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:SYS
-POSTHOOK: query: CREATE DATABASE SYS
+POSTHOOK: query: CREATE DATABASE IF NOT EXISTS SYS
POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:SYS
PREHOOK: query: USE SYS
@@ -1932,35 +1928,21 @@ FROM
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@PART_COL_STATS
POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` (
- `VER_ID` BIGINT,
- `SCHEMA_VERSION` string,
- `VERSION_COMMENT` string,
- CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE
-)
-PREHOOK: type: CREATETABLE
+PREHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.1.0' AS `VERSION_COMMENT`
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: SYS@VERSION
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` (
- `VER_ID` BIGINT,
- `SCHEMA_VERSION` string,
- `VERSION_COMMENT` string,
- CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE
-)
-POSTHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`,
+ 'Hive release version 3.1.0' AS `VERSION_COMMENT`
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: SYS@VERSION
POSTHOOK: Output: database:sys
-PREHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: sys@version
-POSTHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: sys@version
-POSTHOOK: Lineage: version.schema_version SCRIPT []
-POSTHOOK: Lineage: version.ver_id SCRIPT []
-POSTHOOK: Lineage: version.version_comment SCRIPT []
+POSTHOOK: Lineage: VERSION.schema_version SIMPLE []
+POSTHOOK: Lineage: VERSION.ver_id SIMPLE []
+POSTHOOK: Lineage: VERSION.version_comment SIMPLE []
PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` (
`VER_ID` BIGINT,
`SCHEMA_VERSION` string,
@@ -2143,7 +2125,7 @@ FROM
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@KEY_CONSTRAINTS
POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS
SELECT
`TBL_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2157,7 +2139,7 @@ PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@table_params
PREHOOK: Output: SYS@TABLE_STATS_VIEW
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS
SELECT
`TBL_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2178,7 +2160,7 @@ POSTHOOK: Lineage: TABLE_STATS_VIEW.raw_data_size EXPRESSION [(table_params)tabl
POSTHOOK: Lineage: TABLE_STATS_VIEW.tbl_id SIMPLE [(table_params)table_params.FieldSchema(name:tbl_id, type:bigint, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_STATS_VIEW.total_size EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_STATS_VIEW.transient_last_ddl_time EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS
+PREHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS
SELECT
`PART_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2192,7 +2174,7 @@ PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@partition_params
PREHOOK: Output: SYS@PARTITION_STATS_VIEW
PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS
+POSTHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS
SELECT
`PART_ID`,
max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE,
@@ -2469,14 +2451,10 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_MAPPINGS
POSTHOOK: Output: database:sys
-PREHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE
-PREHOOK: type: DROPDATABASE
-POSTHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE
-POSTHOOK: type: DROPDATABASE
-PREHOOK: query: CREATE DATABASE INFORMATION_SCHEMA
+PREHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:INFORMATION_SCHEMA
-POSTHOOK: query: CREATE DATABASE INFORMATION_SCHEMA
+POSTHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA
POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:INFORMATION_SCHEMA
PREHOOK: query: USE INFORMATION_SCHEMA
@@ -2485,7 +2463,7 @@ PREHOOK: Input: database:information_schema
POSTHOOK: query: USE INFORMATION_SCHEMA
POSTHOOK: type: SWITCHDATABASE
POSTHOOK: Input: database:information_schema
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
+PREHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA`
(
`CATALOG_NAME`,
`SCHEMA_NAME`,
@@ -2518,7 +2496,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@SCHEMATA
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
+POSTHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA`
(
`CATALOG_NAME`,
`SCHEMA_NAME`,
@@ -2558,7 +2536,7 @@ POSTHOOK: Lineage: SCHEMATA.default_character_set_schema EXPRESSION []
POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
#### A masked pattern was here ####
POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)d.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLES`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2600,7 +2578,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@TABLES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLES`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2654,7 +2632,7 @@ POSTHOOK: Lineage: TABLES.table_type EXPRESSION [(tbls)t.FieldSchema(name:view_o
POSTHOOK: Lineage: TABLES.user_defined_type_catalog EXPRESSION []
POSTHOOK: Lineage: TABLES.user_defined_type_name EXPRESSION []
POSTHOOK: Lineage: TABLES.user_defined_type_schema EXPRESSION []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
+PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -2693,7 +2671,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@TABLE_PRIVILEGES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -2740,7 +2718,7 @@ POSTHOOK: Lineage: TABLE_PRIVILEGES.table_catalog SIMPLE []
POSTHOOK: Lineage: TABLE_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: TABLE_PRIVILEGES.with_hierarchy SIMPLE []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
+PREHOOK: query: CREATE OR REPLACE VIEW `COLUMNS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -2902,7 +2880,7 @@ PREHOOK: Input: sys@tbl_col_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMNS
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
+POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMNS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -3111,7 +3089,7 @@ POSTHOOK: Lineage: COLUMNS.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, ty
POSTHOOK: Lineage: COLUMNS.udt_catalog EXPRESSION []
POSTHOOK: Lineage: COLUMNS.udt_name EXPRESSION []
POSTHOOK: Lineage: COLUMNS.udt_schema EXPRESSION []
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
+PREHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -3154,7 +3132,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
+POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
`GRANTOR`,
`GRANTEE`,
@@ -3205,7 +3183,7 @@ POSTHOOK: Lineage: COLUMN_PRIVILEGES.privilege_type SIMPLE [(tbl_col_privs)p.Fie
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_catalog SIMPLE []
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
-PREHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
+PREHOOK: query: CREATE OR REPLACE VIEW `VIEWS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
@@ -3247,7 +3225,7 @@ PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@VIEWS
PREHOOK: Output: database:information_schema
-POSTHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
+POSTHOOK: query: CREATE OR REPLACE VIEW `VIEWS`
(
`TABLE_CATALOG`,
`TABLE_SCHEMA`,
http://git-wip-us.apache.org/repos/asf/hive/blob/ee8c72ae/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
index f868982..2da07a5 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -47,7 +47,9 @@ public class HiveSchemaHelper {
public static final String EMBEDDED_HS2_URL =
"jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;"
+ "hive.security.authorization.sqlstd.confwhitelist.append=.*;hive.security.authorization.enabled=false;"
- + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory";
+ + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory;"
+ + "hive.support.concurrency=false;hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager;"
+ + "hive.metastore.rawstore.impl=org.apache.hadoop.hive.metastore.ObjectStore";
public static final String HIVE_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
/***
[5/7] hive git commit: HIVE-19850 : Dynamic partition pruning in Tez
is leading to 'No work found for tablescan' error (Ganesha Shreedhara via
Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-19850 : Dynamic partition pruning in Tez is leading to 'No work found for tablescan' error (Ganesha Shreedhara via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e865b449
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e865b449
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e865b449
Branch: refs/heads/master-txnstats
Commit: e865b449efca04b02922bbc42693a1e8f91825a9
Parents: 5e2a530
Author: Ganesha Shreedhara <ga...@gmail.com>
Authored: Wed Jul 4 23:03:10 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jul 4 23:03:10 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/parse/GenTezUtils.java | 9 +
ql/src/test/queries/clientpositive/dpp.q | 30 ++
.../test/results/clientpositive/llap/dpp.q.out | 371 +++++++++++++++++++
4 files changed, 411 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index cd09481..978a806 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -491,6 +491,7 @@ minillaplocal.query.files=\
cross_prod_1.q,\
cross_prod_3.q,\
cross_prod_4.q,\
+ dpp.q,\
dynamic_partition_pruning.q,\
dynamic_semijoin_reduction.q,\
dynamic_semijoin_reduction_2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index bb0de94..dd1d6a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -276,6 +276,15 @@ public class GenTezUtils {
rsToSemiJoinBranchInfo.put(rs, newSJInfo);
}
}
+ // This TableScanOperator could also be part of other events in eventOperatorSet.
+ for(AppMasterEventOperator event: context.eventOperatorSet) {
+ if(event.getConf() instanceof DynamicPruningEventDesc) {
+ TableScanOperator ts = ((DynamicPruningEventDesc) event.getConf()).getTableScan();
+ if(ts.equals(orig)){
+ ((DynamicPruningEventDesc) event.getConf()).setTableScan((TableScanOperator) newRoot);
+ }
+ }
+ }
}
context.rootToWorkMap.remove(orig);
context.rootToWorkMap.put(newRoot, work);
http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/test/queries/clientpositive/dpp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dpp.q b/ql/src/test/queries/clientpositive/dpp.q
new file mode 100644
index 0000000..4bc7ccb
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dpp.q
@@ -0,0 +1,30 @@
+set hive.tez.dynamic.partition.pruning=true;
+
+CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string);
+
+CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string);
+
+CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string);
+
+
+
+insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true);
+
+insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true);
+
+insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true);
+
+
+
+CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2`;
+
+CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3`;
+
+create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt;
+
+CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1`;
+
+
+explain
+select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt;
+select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt;
http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/test/results/clientpositive/llap/dpp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dpp.q.out b/ql/src/test/results/clientpositive/llap/dpp.q.out
new file mode 100644
index 0000000..240fb4c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/dpp.q.out
@@ -0,0 +1,371 @@
+PREHOOK: query: CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2
+POSTHOOK: query: CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2
+PREHOOK: query: CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t3
+POSTHOOK: query: CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t3
+PREHOOK: query: insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1@dt=2018
+POSTHOOK: query: insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1@dt=2018
+POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_boolean SCRIPT []
+POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_float SCRIPT []
+POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_int SCRIPT []
+POSTHOOK: Lineage: t1 PARTITION(dt=2018).key SCRIPT []
+POSTHOOK: Lineage: t1 PARTITION(dt=2018).value SCRIPT []
+PREHOOK: query: insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t2@dt=2018
+POSTHOOK: query: insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t2@dt=2018
+POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_boolean SCRIPT []
+POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_float SCRIPT []
+POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_int SCRIPT []
+POSTHOOK: Lineage: t2 PARTITION(dt=2018).key SCRIPT []
+POSTHOOK: Lineage: t2 PARTITION(dt=2018).value SCRIPT []
+PREHOOK: query: insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3@dt=2018
+POSTHOOK: query: insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3@dt=2018
+POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_boolean SCRIPT []
+POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_float SCRIPT []
+POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_int SCRIPT []
+POSTHOOK: Lineage: t3 PARTITION(dt=2018).key SCRIPT []
+POSTHOOK: Lineage: t3 PARTITION(dt=2018).value SCRIPT []
+PREHOOK: query: CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2`
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view1
+POSTHOOK: query: CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2`
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view1
+POSTHOOK: Lineage: view1.c_boolean EXPRESSION [(t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), ]
+POSTHOOK: Lineage: view1.c_float EXPRESSION [(t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t2)t2.FieldSchema(name:c_float, type:float, comment:null), ]
+POSTHOOK: Lineage: view1.c_int EXPRESSION [(t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t2)t2.FieldSchema(name:c_int, type:int, comment:null), ]
+POSTHOOK: Lineage: view1.dt EXPRESSION [(t1)t1.FieldSchema(name:dt, type:string, comment:null), (t2)t2.FieldSchema(name:dt, type:string, comment:null), ]
+POSTHOOK: Lineage: view1.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: view1.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3`
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view2
+POSTHOOK: query: CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3`
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view2
+POSTHOOK: Lineage: view2.c_boolean EXPRESSION [(t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), (t3)t3.FieldSchema(name:c_boolean, type:boolean, comment:null), ]
+POSTHOOK: Lineage: view2.c_float EXPRESSION [(t2)t2.FieldSchema(name:c_float, type:float, comment:null), (t3)t3.FieldSchema(name:c_float, type:float, comment:null), ]
+POSTHOOK: Lineage: view2.c_int EXPRESSION [(t2)t2.FieldSchema(name:c_int, type:int, comment:null), (t3)t3.FieldSchema(name:c_int, type:int, comment:null), ]
+POSTHOOK: Lineage: view2.dt EXPRESSION [(t2)t2.FieldSchema(name:dt, type:string, comment:null), (t3)t3.FieldSchema(name:dt, type:string, comment:null), ]
+POSTHOOK: Lineage: view2.key EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), (t3)t3.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: view2.value EXPRESSION [(t2)t2.FieldSchema(name:value, type:string, comment:null), (t3)t3.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2018
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2018
+PREHOOK: Input: default@t3
+PREHOOK: Input: default@t3@dt=2018
+PREHOOK: Input: default@view1
+PREHOOK: Input: default@view2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t4
+POSTHOOK: query: create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2018
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2018
+POSTHOOK: Input: default@t3
+POSTHOOK: Input: default@t3@dt=2018
+POSTHOOK: Input: default@view1
+POSTHOOK: Input: default@view2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t4
+POSTHOOK: Lineage: t4.c_boolean EXPRESSION [(t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), ]
+POSTHOOK: Lineage: t4.c_float EXPRESSION [(t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t2)t2.FieldSchema(name:c_float, type:float, comment:null), ]
+POSTHOOK: Lineage: t4.c_int EXPRESSION [(t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t2)t2.FieldSchema(name:c_int, type:int, comment:null), ]
+POSTHOOK: Lineage: t4.dt EXPRESSION [(t1)t1.FieldSchema(name:dt, type:string, comment:null), (t1)t1.FieldSchema(name:dt, type:string, comment:null), (t2)t2.FieldSchema(name:dt, type:string, comment:null), ]
+POSTHOOK: Lineage: t4.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1`
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view3
+POSTHOOK: query: CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1`
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@view3
+POSTHOOK: Lineage: view3.c_boolean EXPRESSION [(t4)t4.FieldSchema(name:c_boolean, type:boolean, comment:null), (t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), ]
+POSTHOOK: Lineage: view3.c_float EXPRESSION [(t4)t4.FieldSchema(name:c_float, type:float, comment:null), (t1)t1.FieldSchema(name:c_float, type:float, comment:null), ]
+POSTHOOK: Lineage: view3.c_int EXPRESSION [(t4)t4.FieldSchema(name:c_int, type:int, comment:null), (t1)t1.FieldSchema(name:c_int, type:int, comment:null), ]
+POSTHOOK: Lineage: view3.dt EXPRESSION [(t4)t4.FieldSchema(name:dt, type:string, comment:null), (t1)t1.FieldSchema(name:dt, type:string, comment:null), ]
+POSTHOOK: Lineage: view3.key EXPRESSION [(t4)t4.FieldSchema(name:key, type:string, comment:null), (t1)t1.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: view3.value EXPRESSION [(t4)t4.FieldSchema(name:value, type:string, comment:null), (t1)t1.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: explain
+select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Map 5 <- Union 2 (CONTAINS)
+ Map 6 <- Union 7 (CONTAINS)
+ Map 8 <- Union 7 (CONTAINS)
+ Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 7 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ filterExpr: dt is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: dt (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: t3
+ filterExpr: dt is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: dt (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: t4
+ filterExpr: dt is not null (type: boolean)
+ Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: dt is not null (type: boolean)
+ Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: dt (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: dt (string)
+ Target Input: t2
+ Partition key expr: dt
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 1
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: dt (string)
+ Target Input: t3
+ Partition key expr: dt
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 5
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ filterExpr: dt is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: dt (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: dt (string)
+ Target Input: t2
+ Partition key expr: dt
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 1
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: dt (string)
+ Target Input: t3
+ Partition key expr: dt
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 5
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Union 2
+ Vertex: Union 2
+ Union 7
+ Vertex: Union 7
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2018
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2018
+PREHOOK: Input: default@t3
+PREHOOK: Input: default@t3@dt=2018
+PREHOOK: Input: default@t4
+PREHOOK: Input: default@view2
+PREHOOK: Input: default@view3
+#### A masked pattern was here ####
+POSTHOOK: query: select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2018
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2018
+POSTHOOK: Input: default@t3
+POSTHOOK: Input: default@t3@dt=2018
+POSTHOOK: Input: default@t4
+POSTHOOK: Input: default@view2
+POSTHOOK: Input: default@view3
+#### A masked pattern was here ####
+12
[2/7] hive git commit: HIVE-20025: Clean-up of event files created by
HiveProtoLoggingHook (Sankar Hariappan, reviewed by Harish Jaiprakash,
Anishek Agarwal)
Posted by se...@apache.org.
HIVE-20025: Clean-up of event files created by HiveProtoLoggingHook (Sankar Hariappan, reviewed by Harish Jaiprakash, Anishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6311e0b0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6311e0b0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6311e0b0
Branch: refs/heads/master-txnstats
Commit: 6311e0b031b5937d39bde8aad9916c8d0911f0b3
Parents: ee8c72a
Author: Sankar Hariappan <sa...@apache.org>
Authored: Wed Jul 4 13:08:00 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Wed Jul 4 13:08:00 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 11 ++
.../TestHiveProtoEventsCleanerTask.java | 141 ++++++++++++++++
.../metastore/HiveProtoEventsCleanerTask.java | 168 +++++++++++++++++++
.../hive/ql/hooks/HiveProtoLoggingHook.java | 15 +-
.../hive/ql/hooks/TestHiveProtoLoggingHook.java | 2 +-
.../hive/metastore/conf/MetastoreConf.java | 3 +-
6 files changed, 331 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7ef22d6..2da9086 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -625,6 +625,17 @@ public class HiveConf extends Configuration {
"Table alias will be added to column names for queries of type \"select *\" or \n" +
"if query explicitly uses table alias \"select r1.x..\"."),
+ HIVE_PROTO_EVENTS_BASE_PATH("hive.hook.proto.base-directory", "",
+ "Base directory into which the proto event messages are written by HiveProtoLoggingHook."),
+ HIVE_PROTO_EVENTS_QUEUE_CAPACITY("hive.hook.proto.queue.capacity", 64,
+ "Queue capacity for the proto events logging threads."),
+ HIVE_PROTO_EVENTS_CLEAN_FREQ("hive.hook.proto.events.clean.freq", "1d",
+ new TimeValidator(TimeUnit.DAYS),
+ "Frequency at which timer task runs to purge expired proto event files."),
+ HIVE_PROTO_EVENTS_TTL("hive.hook.proto.events.ttl", "7d",
+ new TimeValidator(TimeUnit.DAYS),
+ "Time-To-Live (TTL) of proto event files before cleanup."),
+
// Hadoop Configuration Properties
// Properties with null values are ignored and exist only for the purpose of giving us
// a symbolic name to reference in the Hive source code. Properties with non-null
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveProtoEventsCleanerTask.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveProtoEventsCleanerTask.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveProtoEventsCleanerTask.java
new file mode 100644
index 0000000..e187fad
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveProtoEventsCleanerTask.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestHiveProtoEventsCleanerTask {
+ protected static final Logger LOG = LoggerFactory.getLogger(TestHiveProtoEventsCleanerTask.class);
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+ private Path baseDir;
+ private HiveConf hiveConf;
+ private SystemClock clock = SystemClock.getInstance();
+ private HiveProtoEventsCleanerTask cleanerTask;
+ private FileSystem fs;
+
+ private final String[] eventsSubDirs = new String[] { "query_data", "dag_meta", "dag_data", "app_data" };
+
+ @Before
+ public void setup() throws Exception {
+ hiveConf = new HiveConf(TestHiveProtoEventsCleanerTask.class);
+ String tmpFolder = folder.newFolder().getAbsolutePath();
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_PROTO_EVENTS_BASE_PATH, tmpFolder + "/" + eventsSubDirs[0]);
+ HiveConf.setTimeVar(hiveConf, ConfVars.HIVE_PROTO_EVENTS_TTL, 2, TimeUnit.DAYS);
+
+ baseDir = new Path(tmpFolder);
+ fs = baseDir.getFileSystem(hiveConf);
+ cleanerTask = JavaUtils.newInstance(HiveProtoEventsCleanerTask.class);
+ cleanerTask.setConf(hiveConf);
+ }
+
+ /**
+ * Returns the current date, using the underlying clock in UTC time.
+ */
+ private LocalDate getNow() {
+ // Use UTC date to ensure reader date is same on all timezones.
+ return LocalDateTime.ofEpochSecond(clock.getTime() / 1000, 0, ZoneOffset.UTC).toLocalDate();
+ }
+
+ /**
+ * Returns the directory name for a given date.
+ */
+ public String getDirForDate(LocalDate date) {
+ return "date=" + DateTimeFormatter.ISO_LOCAL_DATE.format(date);
+ }
+
+ private void addDatePartition(Path basePath, LocalDate date) throws Exception {
+ if (!fs.exists(basePath)) {
+ fs.mkdirs(basePath);
+ fs.setPermission(basePath, FsPermission.createImmutable((short)01777));
+ }
+
+ Path datePtn = new Path(basePath, getDirForDate(date));
+ fs.mkdirs(datePtn);
+ fs.setPermission(datePtn, FsPermission.createImmutable((short) 01777));
+ FsPermission.setUMask(hiveConf, FsPermission.createImmutable((short) 0066));
+ Path partFile = new Path(datePtn, "data");
+ FSDataOutputStream out = fs.create(partFile);
+ out.writeInt(1000);
+ out.close();
+ }
+
+ @Test
+ public void testCleanup() throws Exception {
+ int[] inRange = { 3, 5, 2, 1 }; // Must have one entry per eventsSubDirs
+ int[] outRange = { 2, 2, 2, 1 }; // Must have one entry per eventsSubDirs
+ LocalDate today = getNow();
+
+ // Add partitions for the given range of dates from today to past.
+ for (int i = 0; i < inRange.length; i++) {
+ Path basePath = new Path(baseDir + "/" + eventsSubDirs[i]);
+ for (int j = 0; j < inRange[i]; j++) {
+ addDatePartition(basePath, today.minusDays(j));
+ }
+ }
+
+ // Run the task to cleanup
+ cleanerTask.run();
+
+ // Verify if the remaining partitions are not expired ones.
+ String expiredPtn = getDirForDate(today.minusDays(2));
+ for (int i = 0; i < inRange.length; i++) {
+ Path basePath = new Path(baseDir + "/" + eventsSubDirs[i]);
+ FileStatus[] statuses = fs.listStatus(basePath);
+
+ // If the test setup created today and if test runs tomorrow, then extra dir will be deleted.
+ // So, checking for both cases.
+ assertTrue((statuses.length == outRange[i]) || (statuses.length == (outRange[i] - 1)));
+ for (FileStatus status : statuses) {
+ assertTrue(status.getPath().getName().compareTo(expiredPtn) >= 0);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/metastore/src/java/org/apache/hadoop/hive/metastore/HiveProtoEventsCleanerTask.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveProtoEventsCleanerTask.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveProtoEventsCleanerTask.java
new file mode 100644
index 0000000..2a772e2
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveProtoEventsCleanerTask.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class HiveProtoEventsCleanerTask implements MetastoreTaskThread {
+ public static final Logger LOG = LoggerFactory.getLogger(HiveProtoEventsCleanerTask.class);
+
+ private final String[] eventsSubDirs = new String[] { "query_data", "dag_meta", "dag_data", "app_data" };
+ private List<Path> eventsBasePaths = new ArrayList<>();
+ private Configuration conf;
+ private long ttl;
+ private static String expiredDatePtn = null;
+ private static final SystemClock clock = SystemClock.getInstance();
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ String hiveEventsDir = HiveConf.getVar(conf, ConfVars.HIVE_PROTO_EVENTS_BASE_PATH);
+ if (StringUtils.isBlank(hiveEventsDir)) {
+ return;
+ }
+ Path hiveEventsBasePath = new Path(hiveEventsDir);
+ Path baseDir = hiveEventsBasePath.getParent();
+ for (String subDir : eventsSubDirs) {
+ eventsBasePaths.add(new Path(baseDir, subDir));
+ }
+ assert(eventsBasePaths.get(0).equals(hiveEventsBasePath));
+ ttl = HiveConf.getTimeVar(conf, ConfVars.HIVE_PROTO_EVENTS_TTL, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public long runFrequency(TimeUnit unit) {
+ return HiveConf.getTimeVar(conf, ConfVars.HIVE_PROTO_EVENTS_CLEAN_FREQ, unit);
+ }
+
+ @Override
+ public void run() {
+ // If Hive proto logging is not enabled, then nothing to be cleaned-up.
+ if (eventsBasePaths.isEmpty()) {
+ return;
+ }
+
+ // Expired date should be computed each time we run cleaner thread.
+ computeExpiredDatePtn(ttl);
+ for (Path basePath : eventsBasePaths) {
+ cleanupDir(basePath);
+ }
+ }
+
+ /**
+ * Compute the expired date partition, using the underlying clock in UTC time.
+ */
+ private static void computeExpiredDatePtn(long ttl) {
+ // Use UTC date to ensure reader date is same on all timezones.
+ LocalDate expiredDate
+ = LocalDateTime.ofEpochSecond((clock.getTime() - ttl) / 1000, 0, ZoneOffset.UTC).toLocalDate();
+ expiredDatePtn = "date=" + DateTimeFormatter.ISO_LOCAL_DATE.format(expiredDate);
+ }
+
+ /**
+ * Path filters to include only expired date partitions based on TTL.
+ */
+ private static final PathFilter expiredDatePartitionsFilter = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ String dirName = path.getName();
+ return ((dirName.startsWith("date="))
+ && (dirName.compareTo(expiredDatePtn) <= 0));
+ }
+ };
+
+ /**
+ * Finds the expired date partitioned events directory based on TTL and delete them.
+ */
+ private void cleanupDir(Path eventsBasePath) {
+ LOG.debug("Trying to delete expired proto events from " + eventsBasePath);
+ try {
+ FileSystem fs = FileSystem.get(eventsBasePath.toUri(), conf);
+ if (!fs.exists(eventsBasePath)) {
+ return;
+ }
+ FileStatus[] statuses = fs.listStatus(eventsBasePath, expiredDatePartitionsFilter);
+ for (FileStatus dir : statuses) {
+ try {
+ deleteDirByOwner(fs, dir);
+ LOG.info("Deleted expired proto events dir: " + dir.getPath());
+ } catch (IOException ioe) {
+ // Log error and continue to delete other expired dirs.
+ LOG.error("Error deleting expired proto events dir " + dir.getPath(), ioe);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Error while trying to delete expired proto events from " + eventsBasePath, e);
+ }
+ }
+
+ /**
+ * Delete the events dir with it's owner as proxy user.
+ */
+ private void deleteDirByOwner(FileSystem fs, FileStatus eventsDir) throws IOException {
+ String owner = eventsDir.getOwner();
+ if (owner.equals(System.getProperty("user.name"))) {
+ fs.delete(eventsDir.getPath(), true);
+ } else {
+ LOG.info("Deleting " + eventsDir.getPath() + " as user " + owner);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser(owner,
+ UserGroupInformation.getLoginUser());
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ // New FileSystem object to be obtained in user context for doAs flow.
+ try (FileSystem doAsFs = FileSystem.newInstance(eventsDir.getPath().toUri(), conf)) {
+ doAsFs.delete(eventsDir.getPath(), true);
+ }
+ return null;
+ }
+ });
+ } catch (InterruptedException ie) {
+ LOG.error("Could not delete " + eventsDir.getPath() + " for UGI: " + ugi, ie);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
index 0820bea..f463437 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
@@ -102,8 +102,10 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -158,9 +160,7 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext {
.collect(Collectors.toSet());
}
- public static final String HIVE_EVENTS_BASE_PATH = "hive.hook.proto.base-directory";
- public static final String HIVE_HOOK_PROTO_QUEUE_CAPACITY = "hive.hook.proto.queue.capacity";
- public static final int HIVE_HOOK_PROTO_QUEUE_CAPACITY_DEFAULT = 64;
+ private static final int HIVE_HOOK_PROTO_QUEUE_CAPACITY_DEFAULT = 64;
private static final int WAIT_TIME = 5;
public enum EventType {
@@ -190,9 +190,10 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext {
this.clock = clock;
// randomUUID is slow, since its cryptographically secure, only first query will take time.
this.logFileName = "hive_" + UUID.randomUUID().toString();
- String baseDir = conf.get(HIVE_EVENTS_BASE_PATH);
- if (baseDir == null) {
- LOG.error(HIVE_EVENTS_BASE_PATH + " is not set, logging disabled.");
+ String baseDir = conf.getVar(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH);
+ if (StringUtils.isBlank(baseDir)) {
+ baseDir = null;
+ LOG.error(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH.varname + " is not set, logging disabled.");
}
DatePartitionedLogger<HiveHookEventProto> tmpLogger = null;
@@ -211,7 +212,7 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext {
return;
}
- int queueCapacity = conf.getInt(HIVE_HOOK_PROTO_QUEUE_CAPACITY,
+ int queueCapacity = conf.getInt(ConfVars.HIVE_PROTO_EVENTS_QUEUE_CAPACITY.varname,
HIVE_HOOK_PROTO_QUEUE_CAPACITY_DEFAULT);
ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true)
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
index 96fb73c..8124528 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
@@ -64,7 +64,7 @@ public class TestHiveProtoLoggingHook {
public void setup() throws Exception {
conf = new HiveConf();
tmpFolder = folder.newFolder().getAbsolutePath();
- conf.set(HiveProtoLoggingHook.HIVE_EVENTS_BASE_PATH, tmpFolder);
+ conf.setVar(HiveConf.ConfVars.HIVE_PROTO_EVENTS_BASE_PATH, tmpFolder);
QueryState state = new QueryState.Builder().withHiveConf(conf).build();
@SuppressWarnings("serial")
QueryPlan queryPlan = new QueryPlan(HiveOperation.QUERY) {};
http://git-wip-us.apache.org/repos/asf/hive/blob/6311e0b0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 19da432..74a301f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -763,7 +763,8 @@ public class MetastoreConf {
EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," +
"org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," +
MaterializationsCacheCleanerTask.class.getName() + "," +
- MaterializationsRebuildLockCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName(),
+ MaterializationsRebuildLockCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," +
+ "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask",
"Comma separated list of tasks that will be started in separate threads. These will " +
"always be started, regardless of whether the metastore is running in embedded mode " +
"or in server mode. They must implement " + MetastoreTaskThread.class.getName()),
[6/7] hive git commit: HIVE-20018 : fix TestReplicationScenarios on
the branch (Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-20018 : fix TestReplicationScenarios on the branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ffb02e78
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ffb02e78
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ffb02e78
Branch: refs/heads/master-txnstats
Commit: ffb02e7869cedf2b0911ecec9ba8403e189f101c
Parents: 5225cef
Author: sergey <se...@apache.org>
Authored: Thu Jul 5 13:37:12 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Jul 5 13:37:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/InjectableBehaviourObjectStore.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb02e78/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index abbcda3..c5977b2 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore;
import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -135,6 +136,12 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
}
@Override
+ public Table getTable(String catName, String dbName, String tableName,
+ long txnId, String writeIdList) throws MetaException {
+ return getTableModifier.apply(super.getTable(catName, dbName, tableName, txnId, writeIdList));
+ }
+
+ @Override
public Partition getPartition(String catName, String dbName, String tableName,
List<String> partVals) throws NoSuchObjectException, MetaException {
return getPartitionModifier.apply(super.getPartition(catName, dbName, tableName, partVals));
[4/7] hive git commit: HIVE-20066 : hive.load.data.owner is compared
to full principal (Daniel Voros via Zoltan Haindrich)
Posted by se...@apache.org.
HIVE-20066 : hive.load.data.owner is compared to full principal (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e2a530c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e2a530c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e2a530c
Branch: refs/heads/master-txnstats
Commit: 5e2a530cc857c36dc97de6f5bec4003c569d00bd
Parents: 3b6d4e2
Author: Daniel Voros <da...@gmail.com>
Authored: Wed Jul 4 12:15:34 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jul 4 12:15:34 2018 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5e2a530c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index c2ffe02..cd62d49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3879,7 +3879,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
try {
srcs = srcFs.getFileStatus(srcf);
- String runningUser = UserGroupInformation.getLoginUser().getUserName();
+ String runningUser = UserGroupInformation.getLoginUser().getShortUserName();
boolean isOwned = FileUtils.isOwnerOfFileHierarchy(srcFs, srcs, configuredOwner, false);
if (configuredOwner.equals(runningUser)) {
// Check if owner has write permission, else it will have to copy
[7/7] hive git commit: HIVE-19416 : merge master into branch (Sergey
Shelukhin) 0705
Posted by se...@apache.org.
HIVE-19416 : merge master into branch (Sergey Shelukhin) 0705
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f384d4ca
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f384d4ca
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f384d4ca
Branch: refs/heads/master-txnstats
Commit: f384d4cac19a64236fd8fee58d27b2057295df61
Parents: ffb02e7 e865b44
Author: sergey <se...@apache.org>
Authored: Thu Jul 5 13:37:59 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Jul 5 13:37:59 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 11 +
.../TestHiveProtoEventsCleanerTask.java | 141 +++++++
.../test/resources/testconfiguration.properties | 1 +
.../upgrade/hive/hive-schema-3.1.0.hive.sql | 32 +-
.../hive/upgrade-3.0.0-to-3.1.0.hive.sql | 10 +
.../scripts/upgrade/hive/upgrade.order.hive | 1 +
.../metastore/HiveProtoEventsCleanerTask.java | 168 +++++++++
packaging/src/main/assembly/bin.xml | 3 +-
.../hive/ql/hooks/HiveProtoLoggingHook.java | 15 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 9 +
.../hadoop/hive/ql/session/SessionState.java | 2 +-
.../hive/ql/hooks/TestHiveProtoLoggingHook.java | 2 +-
ql/src/test/queries/clientpositive/dpp.q | 30 ++
.../test/results/clientpositive/llap/dpp.q.out | 371 +++++++++++++++++++
.../clientpositive/llap/resourceplan.q.out | 84 ++---
.../llap/strict_managed_tables_sysdb.q.out | 84 ++---
.../hive/metastore/conf/MetastoreConf.java | 3 +-
.../hive/metastore/tools/HiveSchemaHelper.java | 4 +-
19 files changed, 834 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f384d4ca/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
[3/7] hive git commit: HIVE-20016: Investigate
TestJdbcWithMiniHS2.testParallelCompilation3 random failure (Yongzhi Chen,
reviewed by Aihua Xu)
Posted by se...@apache.org.
HIVE-20016: Investigate TestJdbcWithMiniHS2.testParallelCompilation3 random failure (Yongzhi Chen, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3b6d4e27
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3b6d4e27
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3b6d4e27
Branch: refs/heads/master-txnstats
Commit: 3b6d4e279f717ae82a159b33fc858996bf3a3b53
Parents: 6311e0b
Author: Yongzhi Chen <yc...@apache.org>
Authored: Wed Jul 4 07:58:07 2018 -0400
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Wed Jul 4 07:58:07 2018 -0400
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6d4e27/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 6762ee6..27f0216 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -897,7 +897,7 @@ public class SessionState {
/**
* Setup authentication and authorization plugins for this session.
*/
- private void setupAuth() {
+ private synchronized void setupAuth() {
if (authenticator != null) {
// auth has been initialized