You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/05/09 16:07:03 UTC
hive git commit: HIVE-19448: Vectorization: sysdb test doesn't work
after enabling vectorization by default (Matt McCline,
reviewed by Deepak Jaiswal)
Repository: hive
Updated Branches:
refs/heads/master 99a2b8bd6 -> dec0a7ed8
HIVE-19448: Vectorization: sysdb test doesn't work after enabling vectorization by default (Matt McCline, reviewed by Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dec0a7ed
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dec0a7ed
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dec0a7ed
Branch: refs/heads/master
Commit: dec0a7ed8da89f002ee5e7d881fec3c7a7e698a7
Parents: 99a2b8b
Author: Matt McCline <mm...@hortonworks.com>
Authored: Wed May 9 11:06:56 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Wed May 9 11:06:56 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../hive/ql/exec/vector/VectorMapOperator.java | 11 +-
ql/src/test/queries/clientpositive/sysdb.q | 5 +
.../clientpositive/llap/jdbc_handler.q.out | 2 +-
.../results/clientpositive/llap/sysdb.q.out | 526 ++++++++++++-------
5 files changed, 359 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dec0a7ed/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 88a7cfc..fa3f788 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3536,7 +3536,7 @@ public class HiveConf extends Configuration {
"The default value is false."),
HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES(
"hive.vectorized.row.serde.inputformat.excludes",
- "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+ "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat,org.apache.hive.storage.jdbc.JdbcInputFormat",
"The input formats not supported by row deserialize vectorization."),
HIVE_VECTOR_ADAPTOR_USAGE_MODE("hive.vectorized.adaptor.usage.mode", "all", new StringSet("none", "chosen", "all"),
"Specifies the extent to which the VectorUDFAdaptor will be used for UDFs that do not have a corresponding vectorized class.\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/dec0a7ed/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
index 6f1346d..2542e03 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
@@ -945,8 +945,15 @@ public class VectorMapOperator extends AbstractMapOperator {
// Convert input row to standard objects.
List<Object> standardObjects = new ArrayList<Object>();
- ObjectInspectorUtils.copyToStandardObject(standardObjects, deserialized,
- currentPartRawRowObjectInspector, ObjectInspectorCopyOption.WRITABLE);
+ try {
+ ObjectInspectorUtils.copyToStandardObject(
+ standardObjects,
+ deserialized,
+ currentPartRawRowObjectInspector,
+ ObjectInspectorCopyOption.WRITABLE);
+ } catch (Exception e) {
+ throw new HiveException("copyToStandardObject failed: " + e);
+ }
if (standardObjects.size() < currentDataColumnCount) {
throw new HiveException("Input File Format returned row with too few columns");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dec0a7ed/ql/src/test/queries/clientpositive/sysdb.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/sysdb.q b/ql/src/test/queries/clientpositive/sysdb.q
index 1dfcbce..f291354 100644
--- a/ql/src/test/queries/clientpositive/sysdb.q
+++ b/ql/src/test/queries/clientpositive/sysdb.q
@@ -8,6 +8,8 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.cbo.enable=false;
+-- SORT_QUERY_RESULTS
+
create table src_buck (key int, value string) clustered by(value) into 2 buckets;
create table src_skew (key int) skewed by (key) on (1,2,3);
@@ -60,6 +62,9 @@ select role_name from roles order by role_name limit 5;
select principal_name, grantor from role_map order by principal_name, grantor limit 5;
+explain vectorization detail
+select count(*) from sds;
+
select count(*) from sds;
select param_key, param_value from sd_params order by param_key, param_value limit 5;
http://git-wip-us.apache.org/repos/asf/hive/blob/dec0a7ed/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
index e7e3bc3..ffeafdb 100644
--- a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
+++ b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
@@ -237,7 +237,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Execution mode: vectorized, llap
+ Execution mode: llap
LLAP IO: no inputs
Map 4
Map Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/dec0a7ed/ql/src/test/results/clientpositive/llap/sysdb.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index de20f2d..3164637 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -273,7 +273,7 @@ TBLPROPERTIES (
\"OWNER_NAME\",
\"OWNER_TYPE\"
FROM
- DBS"
+ \"DBS\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@DBS
@@ -297,7 +297,7 @@ TBLPROPERTIES (
\"OWNER_NAME\",
\"OWNER_TYPE\"
FROM
- DBS"
+ \"DBS\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@DBS
@@ -1370,7 +1370,7 @@ TBLPROPERTIES (
\"VIEW_EXPANDED_TEXT\",
\"VIEW_ORIGINAL_TEXT\",
\"IS_REWRITE_ENABLED\"
-FROM TBLS"
+FROM \"TBLS\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@TBLS
@@ -1407,7 +1407,7 @@ TBLPROPERTIES (
\"VIEW_EXPANDED_TEXT\",
\"VIEW_ORIGINAL_TEXT\",
\"IS_REWRITE_ENABLED\"
-FROM TBLS"
+FROM \"TBLS\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@TBLS
@@ -1428,7 +1428,7 @@ TBLPROPERTIES (
\"DB_NAME\",
\"TBL_NAME\",
\"TXN_LIST\"
-FROM MV_CREATION_METADATA"
+FROM \"MV_CREATION_METADATA\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@MV_CREATION_METADATA
@@ -1449,7 +1449,7 @@ TBLPROPERTIES (
\"DB_NAME\",
\"TBL_NAME\",
\"TXN_LIST\"
-FROM MV_CREATION_METADATA"
+FROM \"MV_CREATION_METADATA\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@MV_CREATION_METADATA
@@ -1465,7 +1465,7 @@ TBLPROPERTIES (
"SELECT
\"MV_CREATION_METADATA_ID\",
\"TBL_ID\"
-FROM MV_TABLES_USED"
+FROM \"MV_TABLES_USED\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@MV_TABLES_USED
@@ -1481,7 +1481,7 @@ TBLPROPERTIES (
"SELECT
\"MV_CREATION_METADATA_ID\",
\"TBL_ID\"
-FROM MV_TABLES_USED"
+FROM \"MV_TABLES_USED\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@MV_TABLES_USED
@@ -2164,16 +2164,16 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- r.NAME RP_NAME,
- t.NAME NAME,
- TRIGGER_EXPRESSION,
- ACTION_EXPRESSION
+ r.\"NAME\" AS RP_NAME,
+ t.\"NAME\" AS NAME,
+ \"TRIGGER_EXPRESSION\",
+ \"ACTION_EXPRESSION\"
FROM
- WM_TRIGGER t
+ \"WM_TRIGGER\" t
JOIN
- WM_RESOURCEPLAN r
+ \"WM_RESOURCEPLAN\" r
ON
- t.RP_ID = r.RP_ID"
+ t.\"RP_ID\" = r.\"RP_ID\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@WM_TRIGGERS
@@ -2189,16 +2189,16 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- r.NAME RP_NAME,
- t.NAME NAME,
- TRIGGER_EXPRESSION,
- ACTION_EXPRESSION
+ r.\"NAME\" AS RP_NAME,
+ t.\"NAME\" AS NAME,
+ \"TRIGGER_EXPRESSION\",
+ \"ACTION_EXPRESSION\"
FROM
- WM_TRIGGER t
+ \"WM_TRIGGER\" t
JOIN
- WM_RESOURCEPLAN r
+ \"WM_RESOURCEPLAN\" r
ON
- t.RP_ID = r.RP_ID"
+ t.\"RP_ID\" = r.\"RP_ID\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_TRIGGERS
@@ -2215,17 +2215,17 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME,
- WM_POOL.PATH,
- WM_POOL.ALLOC_FRACTION,
- WM_POOL.QUERY_PARALLELISM,
- WM_POOL.SCHEDULING_POLICY
+ \"WM_RESOURCEPLAN\".\"NAME\",
+ \"WM_POOL\".\"PATH\",
+ \"WM_POOL\".\"ALLOC_FRACTION\",
+ \"WM_POOL\".\"QUERY_PARALLELISM\",
+ \"WM_POOL\".\"SCHEDULING_POLICY\"
FROM
- WM_POOL
+ \"WM_POOL\"
JOIN
- WM_RESOURCEPLAN
+ \"WM_RESOURCEPLAN\"
ON
- WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID"
+ \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\""
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@WM_POOLS
@@ -2242,17 +2242,17 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME,
- WM_POOL.PATH,
- WM_POOL.ALLOC_FRACTION,
- WM_POOL.QUERY_PARALLELISM,
- WM_POOL.SCHEDULING_POLICY
+ \"WM_RESOURCEPLAN\".\"NAME\",
+ \"WM_POOL\".\"PATH\",
+ \"WM_POOL\".\"ALLOC_FRACTION\",
+ \"WM_POOL\".\"QUERY_PARALLELISM\",
+ \"WM_POOL\".\"SCHEDULING_POLICY\"
FROM
- WM_POOL
+ \"WM_POOL\"
JOIN
- WM_RESOURCEPLAN
+ \"WM_RESOURCEPLAN\"
ON
- WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID"
+ \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\""
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_POOLS
@@ -2267,21 +2267,21 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME RP_NAME,
- WM_POOL.PATH POOL_PATH,
- WM_TRIGGER.NAME TRIGGER_NAME
-FROM WM_POOL_TO_TRIGGER
- JOIN WM_POOL ON WM_POOL_TO_TRIGGER.POOL_ID = WM_POOL.POOL_ID
- JOIN WM_TRIGGER ON WM_POOL_TO_TRIGGER.TRIGGER_ID = WM_TRIGGER.TRIGGER_ID
- JOIN WM_RESOURCEPLAN ON WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID
+ \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME,
+ \"WM_POOL\".\"PATH\" AS POOL_PATH,
+ \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME
+FROM \"WM_POOL_TO_TRIGGER\"
+ JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\"
+ JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\"
+ JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
UNION
SELECT
- WM_RESOURCEPLAN.NAME RP_NAME,
- '<unmanaged queries>' POOL_PATH,
- WM_TRIGGER.NAME TRIGGER_NAME
-FROM WM_TRIGGER
- JOIN WM_RESOURCEPLAN ON WM_TRIGGER.RP_ID = WM_RESOURCEPLAN.RP_ID
-WHERE WM_TRIGGER.IS_IN_UNMANAGED = 1
+ \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME,
+ '<unmanaged queries>' AS POOL_PATH,
+ \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME
+FROM \"WM_TRIGGER\"
+ JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
+WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't')
"
)
PREHOOK: type: CREATETABLE
@@ -2297,21 +2297,21 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME RP_NAME,
- WM_POOL.PATH POOL_PATH,
- WM_TRIGGER.NAME TRIGGER_NAME
-FROM WM_POOL_TO_TRIGGER
- JOIN WM_POOL ON WM_POOL_TO_TRIGGER.POOL_ID = WM_POOL.POOL_ID
- JOIN WM_TRIGGER ON WM_POOL_TO_TRIGGER.TRIGGER_ID = WM_TRIGGER.TRIGGER_ID
- JOIN WM_RESOURCEPLAN ON WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID
+ \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME,
+ \"WM_POOL\".\"PATH\" AS POOL_PATH,
+ \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME
+FROM \"WM_POOL_TO_TRIGGER\"
+ JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\"
+ JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\"
+ JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
UNION
SELECT
- WM_RESOURCEPLAN.NAME RP_NAME,
- '<unmanaged queries>' POOL_PATH,
- WM_TRIGGER.NAME TRIGGER_NAME
-FROM WM_TRIGGER
- JOIN WM_RESOURCEPLAN ON WM_TRIGGER.RP_ID = WM_RESOURCEPLAN.RP_ID
-WHERE WM_TRIGGER.IS_IN_UNMANAGED = 1
+ \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME,
+ '<unmanaged queries>' AS POOL_PATH,
+ \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME
+FROM \"WM_TRIGGER\"
+ JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
+WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't')
"
)
POSTHOOK: type: CREATETABLE
@@ -2329,14 +2329,14 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME,
- ENTITY_TYPE,
- ENTITY_NAME,
- case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
- ORDERING
-FROM WM_MAPPING
-JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+ \"WM_RESOURCEPLAN\".\"NAME\",
+ \"ENTITY_TYPE\",
+ \"ENTITY_NAME\",
+ case when \"WM_POOL\".\"PATH\" is null then '<unmanaged>' else \"WM_POOL\".\"PATH\" end,
+ \"ORDERING\"
+FROM \"WM_MAPPING\"
+JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
+LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\"
"
)
PREHOOK: type: CREATETABLE
@@ -2354,14 +2354,14 @@ TBLPROPERTIES (
"hive.sql.database.type" = "METASTORE",
"hive.sql.query" =
"SELECT
- WM_RESOURCEPLAN.NAME,
- ENTITY_TYPE,
- ENTITY_NAME,
- case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
- ORDERING
-FROM WM_MAPPING
-JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+ \"WM_RESOURCEPLAN\".\"NAME\",
+ \"ENTITY_TYPE\",
+ \"ENTITY_NAME\",
+ case when \"WM_POOL\".\"PATH\" is null then '<unmanaged>' else \"WM_POOL\".\"PATH\" end,
+ \"ORDERING\"
+FROM \"WM_MAPPING\"
+JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"
+LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\"
"
)
POSTHOOK: type: CREATETABLE
@@ -2393,18 +2393,26 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
`DEFAULT_CHARACTER_SET_NAME`,
`SQL_PATH`
) AS
-SELECT
+SELECT DISTINCT
'default',
- `NAME`,
- `OWNER_NAME`,
+ D.`NAME`,
+ D.`OWNER_NAME`,
cast(null as string),
cast(null as string),
cast(null as string),
`DB_LOCATION_URI`
FROM
- sys.DBS
+ `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P
+WHERE
+ NOT restrict_information_schema() OR
+ D.`DB_ID` = T.`DB_ID`
+ AND T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
+PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@SCHEMATA
PREHOOK: Output: database:information_schema
POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
@@ -2417,27 +2425,35 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA`
`DEFAULT_CHARACTER_SET_NAME`,
`SQL_PATH`
) AS
-SELECT
+SELECT DISTINCT
'default',
- `NAME`,
- `OWNER_NAME`,
+ D.`NAME`,
+ D.`OWNER_NAME`,
cast(null as string),
cast(null as string),
cast(null as string),
`DB_LOCATION_URI`
FROM
- sys.DBS
+ `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P
+WHERE
+ NOT restrict_information_schema() OR
+ D.`DB_ID` = T.`DB_ID`
+ AND T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
+POSTHOOK: Input: sys@tbls
POSTHOOK: Output: INFORMATION_SCHEMA@SCHEMATA
POSTHOOK: Output: database:information_schema
POSTHOOK: Lineage: SCHEMATA.catalog_name SIMPLE []
POSTHOOK: Lineage: SCHEMATA.default_character_set_catalog EXPRESSION []
POSTHOOK: Lineage: SCHEMATA.default_character_set_name EXPRESSION []
POSTHOOK: Lineage: SCHEMATA.default_character_set_schema EXPRESSION []
-POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)dbs.FieldSchema(name:name, type:string, comment:from deserializer), ]
+POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ]
#### A masked pattern was here ####
-POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)dbs.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ]
+POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)d.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ]
PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
(
`TABLE_CATALOG`,
@@ -2453,7 +2469,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
`IS_TYPED`,
`COMMIT_ACTION`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -2467,11 +2483,16 @@ SELECT
'NO',
cast(null as string)
FROM
- `sys`.`TBLS` T, `sys`.`DBS` D
+ `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P
WHERE
D.`DB_ID` = T.`DB_ID`
+ AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_PRIV`='SELECT')
PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@TABLES
PREHOOK: Output: database:information_schema
@@ -2490,7 +2511,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES`
`IS_TYPED`,
`COMMIT_ACTION`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -2504,11 +2525,16 @@ SELECT
'NO',
cast(null as string)
FROM
- `sys`.`TBLS` T, `sys`.`DBS` D
+ `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P
WHERE
D.`DB_ID` = T.`DB_ID`
+ AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_PRIV`='SELECT')
POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
POSTHOOK: Output: INFORMATION_SCHEMA@TABLES
POSTHOOK: Output: database:information_schema
@@ -2535,9 +2561,9 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
`IS_GRANTABLE`,
`WITH_HIERARCHY`
) AS
-SELECT
- `GRANTOR`,
- `PRINCIPAL_NAME`,
+SELECT DISTINCT
+ P.`GRANTOR`,
+ P.`PRINCIPAL_NAME`,
'default',
D.`NAME`,
T.`TBL_NAME`,
@@ -2545,12 +2571,18 @@ SELECT
IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'),
'NO'
FROM
- sys.`TBL_PRIVS` P,
- sys.`TBLS` T,
- sys.`DBS` D
+ `sys`.`TBL_PRIVS` P,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`TBL_PRIVS` P2
WHERE
- P.TBL_ID = T.TBL_ID
- AND T.DB_ID = D.DB_ID
+ P.`TBL_ID` = T.`TBL_ID`
+ AND T.`DB_ID` = D.`DB_ID`
+ AND (NOT restrict_information_schema() OR
+ P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE`
+ AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))
+ AND P2.`TBL_PRIV`='SELECT')
PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@dbs
PREHOOK: Input: sys@tbl_privs
@@ -2568,9 +2600,9 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES`
`IS_GRANTABLE`,
`WITH_HIERARCHY`
) AS
-SELECT
- `GRANTOR`,
- `PRINCIPAL_NAME`,
+SELECT DISTINCT
+ P.`GRANTOR`,
+ P.`PRINCIPAL_NAME`,
'default',
D.`NAME`,
T.`TBL_NAME`,
@@ -2578,12 +2610,18 @@ SELECT
IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'),
'NO'
FROM
- sys.`TBL_PRIVS` P,
- sys.`TBLS` T,
- sys.`DBS` D
+ `sys`.`TBL_PRIVS` P,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`TBL_PRIVS` P2
WHERE
- P.TBL_ID = T.TBL_ID
- AND T.DB_ID = D.DB_ID
+ P.`TBL_ID` = T.`TBL_ID`
+ AND T.`DB_ID` = D.`DB_ID`
+ AND (NOT restrict_information_schema() OR
+ P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE`
+ AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))
+ AND P2.`TBL_PRIV`='SELECT')
POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: sys@dbs
POSTHOOK: Input: sys@tbl_privs
@@ -2648,7 +2686,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
`DECLARED_NUMERIC_PRECISION`,
`DECLARED_NUMERIC_SCALE`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -2737,18 +2775,26 @@ SELECT
WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10
ELSE null END
FROM
- sys.`COLUMNS_V2` C,
- sys.`SDS` S,
- sys.`TBLS` T,
- sys.`DBS` D
+ `sys`.`COLUMNS_V2` C,
+ `sys`.`SDS` S,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`TBL_COL_PRIVS` P
WHERE
S.`SD_ID` = T.`SD_ID`
AND T.`DB_ID` = D.`DB_ID`
AND C.`CD_ID` = S.`CD_ID`
+ AND (NOT restrict_information_schema() OR
+ T.`TBL_ID` = P.`TBL_ID`
+ AND C.`COLUMN_NAME` = P.`COLUMN_NAME`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_COL_PRIV`='SELECT')
PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@columns_v2
PREHOOK: Input: sys@dbs
PREHOOK: Input: sys@sds
+PREHOOK: Input: sys@tbl_col_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMNS
PREHOOK: Output: database:information_schema
@@ -2802,7 +2848,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS`
`DECLARED_NUMERIC_PRECISION`,
`DECLARED_NUMERIC_SCALE`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -2891,18 +2937,26 @@ SELECT
WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10
ELSE null END
FROM
- sys.`COLUMNS_V2` C,
- sys.`SDS` S,
- sys.`TBLS` T,
- sys.`DBS` D
+ `sys`.`COLUMNS_V2` C,
+ `sys`.`SDS` S,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`TBL_COL_PRIVS` P
WHERE
S.`SD_ID` = T.`SD_ID`
AND T.`DB_ID` = D.`DB_ID`
AND C.`CD_ID` = S.`CD_ID`
+ AND (NOT restrict_information_schema() OR
+ T.`TBL_ID` = P.`TBL_ID`
+ AND C.`COLUMN_NAME` = P.`COLUMN_NAME`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_COL_PRIV`='SELECT')
POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: sys@columns_v2
POSTHOOK: Input: sys@dbs
POSTHOOK: Input: sys@sds
+POSTHOOK: Input: sys@tbl_col_privs
POSTHOOK: Input: sys@tbls
POSTHOOK: Output: INFORMATION_SCHEMA@COLUMNS
POSTHOOK: Output: database:information_schema
@@ -2964,32 +3018,35 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
`PRIVILEGE_TYPE`,
`IS_GRANTABLE`
) AS
-SELECT
- `GRANTOR`,
- `PRINCIPAL_NAME`,
+SELECT DISTINCT
+ P.`GRANTOR`,
+ P.`PRINCIPAL_NAME`,
'default',
D.`NAME`,
T.`TBL_NAME`,
- C.`COLUMN_NAME`,
+ P.`COLUMN_NAME`,
P.`TBL_COL_PRIV`,
IF (P.`GRANT_OPTION` == 0, 'NO', 'YES')
FROM
- sys.`TBL_COL_PRIVS` P,
- sys.`TBLS` T,
- sys.`DBS` D,
- sys.`COLUMNS_V2` C,
- sys.`SDS` S
+ `sys`.`TBL_COL_PRIVS` P,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`SDS` S,
+ `sys`.`TBL_PRIVS` P2
WHERE
S.`SD_ID` = T.`SD_ID`
AND T.`DB_ID` = D.`DB_ID`
AND P.`TBL_ID` = T.`TBL_ID`
- AND P.`COLUMN_NAME` = C.`COLUMN_NAME`
- AND C.`CD_ID` = S.`CD_ID`
+ AND (NOT restrict_information_schema() OR
+ P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE`
+ AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))
+ AND P2.`TBL_PRIV`='SELECT')
PREHOOK: type: CREATEVIEW
-PREHOOK: Input: sys@columns_v2
PREHOOK: Input: sys@dbs
PREHOOK: Input: sys@sds
PREHOOK: Input: sys@tbl_col_privs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES
PREHOOK: Output: database:information_schema
@@ -3004,36 +3061,39 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES`
`PRIVILEGE_TYPE`,
`IS_GRANTABLE`
) AS
-SELECT
- `GRANTOR`,
- `PRINCIPAL_NAME`,
+SELECT DISTINCT
+ P.`GRANTOR`,
+ P.`PRINCIPAL_NAME`,
'default',
D.`NAME`,
T.`TBL_NAME`,
- C.`COLUMN_NAME`,
+ P.`COLUMN_NAME`,
P.`TBL_COL_PRIV`,
IF (P.`GRANT_OPTION` == 0, 'NO', 'YES')
FROM
- sys.`TBL_COL_PRIVS` P,
- sys.`TBLS` T,
- sys.`DBS` D,
- sys.`COLUMNS_V2` C,
- sys.`SDS` S
+ `sys`.`TBL_COL_PRIVS` P,
+ `sys`.`TBLS` T,
+ `sys`.`DBS` D,
+ `sys`.`SDS` S,
+ `sys`.`TBL_PRIVS` P2
WHERE
S.`SD_ID` = T.`SD_ID`
AND T.`DB_ID` = D.`DB_ID`
AND P.`TBL_ID` = T.`TBL_ID`
- AND P.`COLUMN_NAME` = C.`COLUMN_NAME`
- AND C.`CD_ID` = S.`CD_ID`
+ AND (NOT restrict_information_schema() OR
+ P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE`
+ AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))
+ AND P2.`TBL_PRIV`='SELECT')
POSTHOOK: type: CREATEVIEW
-POSTHOOK: Input: sys@columns_v2
POSTHOOK: Input: sys@dbs
POSTHOOK: Input: sys@sds
POSTHOOK: Input: sys@tbl_col_privs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
POSTHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES
POSTHOOK: Output: database:information_schema
-POSTHOOK: Lineage: COLUMN_PRIVILEGES.column_name SIMPLE [(columns_v2)c.FieldSchema(name:column_name, type:string, comment:from deserializer), ]
+POSTHOOK: Lineage: COLUMN_PRIVILEGES.column_name SIMPLE [(tbl_col_privs)p.FieldSchema(name:column_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: COLUMN_PRIVILEGES.grantee SIMPLE [(tbl_col_privs)p.FieldSchema(name:principal_name, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: COLUMN_PRIVILEGES.grantor SIMPLE [(tbl_col_privs)p.FieldSchema(name:grantor, type:string, comment:from deserializer), ]
POSTHOOK: Lineage: COLUMN_PRIVILEGES.is_grantable EXPRESSION [(tbl_col_privs)p.FieldSchema(name:grant_option, type:int, comment:from deserializer), ]
@@ -3054,7 +3114,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
`IS_TRIGGER_DELETABLE`,
`IS_TRIGGER_INSERTABLE_INTO`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -3067,12 +3127,19 @@ SELECT
false
FROM
`sys`.`DBS` D,
- `sys`.`TBLS` T
+ `sys`.`TBLS` T,
+ `sys`.`TBL_PRIVS` P
WHERE
- D.`DB_ID` = T.`DB_ID` AND
- length(T.VIEW_ORIGINAL_TEXT) > 0
+ D.`DB_ID` = T.`DB_ID`
+ AND length(T.VIEW_ORIGINAL_TEXT) > 0
+ AND (NOT restrict_information_schema() OR
+ T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_PRIV`='SELECT')
PREHOOK: type: CREATEVIEW
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
PREHOOK: Output: INFORMATION_SCHEMA@VIEWS
PREHOOK: Output: database:information_schema
@@ -3089,7 +3156,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS`
`IS_TRIGGER_DELETABLE`,
`IS_TRIGGER_INSERTABLE_INTO`
) AS
-SELECT
+SELECT DISTINCT
'default',
D.NAME,
T.TBL_NAME,
@@ -3102,12 +3169,19 @@ SELECT
false
FROM
`sys`.`DBS` D,
- `sys`.`TBLS` T
+ `sys`.`TBLS` T,
+ `sys`.`TBL_PRIVS` P
WHERE
- D.`DB_ID` = T.`DB_ID` AND
- length(T.VIEW_ORIGINAL_TEXT) > 0
+ D.`DB_ID` = T.`DB_ID`
+ AND length(T.VIEW_ORIGINAL_TEXT) > 0
+ AND (NOT restrict_information_schema() OR
+ T.`TBL_ID` = P.`TBL_ID`
+ AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
+ OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
+ AND P.`TBL_PRIV`='SELECT')
POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
POSTHOOK: Output: INFORMATION_SCHEMA@VIEWS
POSTHOOK: Output: database:information_schema
@@ -3256,6 +3330,95 @@ POSTHOOK: query: select principal_name, grantor from role_map order by principal
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@role_map
#### A masked pattern was here ####
+PREHOOK: query: explain vectorization detail
+select count(*) from sds
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization detail
+select count(*) from sds
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: sds
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map Vectorization:
+ enabled: false
+ enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS true AND hive.vectorized.row.serde.inputformat.excludes NOT CONTAINS org.apache.hive.storage.jdbc.JdbcInputFormat IS false, hive.vectorized.use.row.serde.deserialize IS false
+ inputFileFormats: org.apache.hive.storage.jdbc.JdbcInputFormat
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder:
+ reduceColumnSortOrder:
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 1
+ dataColumns: VALUE._col0:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ native: false
+ vectorProcessingMode: GLOBAL
+ projectedOutputColumnNums: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select count(*) from sds
PREHOOK: type: QUERY
PREHOOK: Input: sys@sds
@@ -3283,9 +3446,9 @@ POSTHOOK: Input: sys@sequence_table
#### A masked pattern was here ####
org.apache.hadoop.hive.metastore.model.MCatalog
org.apache.hadoop.hive.metastore.model.MColumnDescriptor
-org.apache.hadoop.hive.metastore.model.MCreationMetadata
org.apache.hadoop.hive.metastore.model.MDatabase
org.apache.hadoop.hive.metastore.model.MFunction
+org.apache.hadoop.hive.metastore.model.MGlobalPrivilege
PREHOOK: query: select name, slib from serdes order by name, slib limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@serdes
@@ -3421,8 +3584,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@tab_col_stats
#### A masked pattern was here ####
alltypesorc cbigint 3115 6064
-alltypesorc cboolean1 3114 NULL
-alltypesorc cboolean2 3115 NULL
+alltypesorc cboolean1 3114 0
+alltypesorc cboolean2 3115 0
alltypesorc cdouble 3114 5569
alltypesorc cfloat 3115 131
alltypesorc cint 3115 6029
@@ -3479,11 +3642,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@table_params
POSTHOOK: Input: sys@table_stats_view
#### A masked pattern was here ####
-{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} 0 0 0 0
-{"BASIC_STATS":"true","COLUMN_STATS":{"pool_path":"true","rp_name":"true","trigger_name":"true"}} 0 0 0 0
-{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} 0 0 0 0
-{"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true","c":"true","d":"true","e":"true","f":"true","g":"true"}} 0 0 0 0
-{"BASIC_STATS":"true","COLUMN_STATS":{"create_time":"true","last_access_time":"true","part_id":"true","part_name":"true","sd_id":"true","tbl_id":"true"}} 0 0 0 0
+{"BASIC_STATS":"true","COLUMN_STATS":{"action_expression":"true","name":"true","rp_name":"true","trigger_expression":"true"}} 0 0 0 0
+#### A masked pattern was here ####
+{"BASIC_STATS":"true","COLUMN_STATS":{"db_id":"true","param_key":"true","param_value":"true"}} 0 0 0 0
+{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}} 0 0 0 0
+{"BASIC_STATS":"true","COLUMN_STATS":{"param_key":"true","param_value":"true","serde_id":"true"}} 0 0 0 0
PREHOOK: query: select COLUMN_STATS_ACCURATE, NUM_FILES, NUM_ROWS, RAW_DATA_SIZE, TOTAL_SIZE FROM PARTITION_STATS_VIEW where COLUMN_STATS_ACCURATE is not null order by NUM_FILES, NUM_ROWS, RAW_DATA_SIZE limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@partition_params
@@ -3556,7 +3719,7 @@ STAGE PLANS:
Execution mode: llap
LLAP IO: no inputs
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0)
@@ -3592,27 +3755,35 @@ PREHOOK: Input: database:information_schema
POSTHOOK: query: use INFORMATION_SCHEMA
POSTHOOK: type: SWITCHDATABASE
POSTHOOK: Input: database:information_schema
+Warning: Shuffle Join MERGEJOIN[20][tables = [d, t, p]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select count(*) from SCHEMATA
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@schemata
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
+PREHOOK: Input: sys@tbls
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from SCHEMATA
POSTHOOK: type: QUERY
POSTHOOK: Input: information_schema@schemata
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
+POSTHOOK: Input: sys@tbls
#### A masked pattern was here ####
3
+Warning: Shuffle Join MERGEJOIN[38][tables = [t, d, p]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: select * from TABLES order by TABLE_SCHEMA, TABLE_NAME
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@tables
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
#### A masked pattern was here ####
POSTHOOK: query: select * from TABLES order by TABLE_SCHEMA, TABLE_NAME
POSTHOOK: type: QUERY
POSTHOOK: Input: information_schema@tables
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
#### A masked pattern was here ####
default default alltypesorc BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
@@ -3672,6 +3843,7 @@ default sys wm_pools BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys wm_pools_to_triggers BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys wm_resourceplans BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys wm_triggers BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
+Warning: Shuffle Join MERGEJOIN[60][tables = [p, t, d, p2]] in Stage 'Reducer 4' is a cross product
PREHOOK: query: select * from TABLE_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, PRIVILEGE_TYPE limit 10
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@table_privileges
@@ -3696,12 +3868,14 @@ hive_test_user hive_test_user default default moretypes SELECT YES NO
hive_test_user hive_test_user default default moretypes UPDATE YES NO
hive_test_user hive_test_user default default scr_txn DELETE YES NO
hive_test_user hive_test_user default default scr_txn INSERT YES NO
+Warning: Shuffle Join MERGEJOIN[82][tables = [c, s, t, d, p]] in Stage 'Reducer 5' is a cross product
PREHOOK: query: select * from COLUMNS where TABLE_NAME = 'alltypesorc' or TABLE_NAME = 'moretypes' order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@columns
PREHOOK: Input: sys@columns_v2
PREHOOK: Input: sys@dbs
PREHOOK: Input: sys@sds
+PREHOOK: Input: sys@tbl_col_privs
PREHOOK: Input: sys@tbls
#### A masked pattern was here ####
POSTHOOK: query: select * from COLUMNS where TABLE_NAME = 'alltypesorc' or TABLE_NAME = 'moretypes' order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION
@@ -3710,55 +3884,41 @@ POSTHOOK: Input: information_schema@columns
POSTHOOK: Input: sys@columns_v2
POSTHOOK: Input: sys@dbs
POSTHOOK: Input: sys@sds
+POSTHOOK: Input: sys@tbl_col_privs
POSTHOOK: Input: sys@tbls
#### A masked pattern was here ####
-default default alltypesorc ctinyint 0 NULL YES tinyint NULL NULL 3 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES tinyint 3 10
-default default alltypesorc csmallint 1 NULL YES smallint NULL NULL 5 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES smallint 5 10
-default default alltypesorc cint 2 NULL YES int NULL NULL 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES int 10 10
-default default alltypesorc cbigint 3 NULL YES bigint NULL NULL 19 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES bigint 19 10
-default default alltypesorc cfloat 4 NULL YES float NULL NULL 23 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES float 23 2
-default default alltypesorc cdouble 5 NULL YES double NULL NULL 53 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES double 53 2
-default default alltypesorc cstring1 6 NULL YES string NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES string NULL NULL
-default default alltypesorc cstring2 7 NULL YES string NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES string NULL NULL
-default default alltypesorc ctimestamp1 8 NULL YES timestamp NULL NULL NULL NULL NULL 9 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES timestamp NULL NULL
-default default alltypesorc ctimestamp2 9 NULL YES timestamp NULL NULL NULL NULL NULL 9 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES timestamp NULL NULL
-default default alltypesorc cboolean1 10 NULL YES boolean NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES boolean NULL NULL
-default default alltypesorc cboolean2 11 NULL YES boolean NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES boolean NULL NULL
-default default moretypes a 0 NULL YES decimal(10,2) NULL NULL 10 10 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES decimal(10,2) 10 10
-default default moretypes b 1 NULL YES tinyint NULL NULL 3 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES tinyint 3 10
-default default moretypes c 2 NULL YES smallint NULL NULL 5 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES smallint 5 10
-default default moretypes d 3 NULL YES int NULL NULL 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES int 10 10
-default default moretypes e 4 NULL YES bigint NULL NULL 19 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES bigint 19 10
-default default moretypes f 5 NULL YES varchar(10) 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES varchar(10) NULL NULL
-default default moretypes g 6 NULL YES char(3) 3 3 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES char(3) NULL NULL
+Warning: Shuffle Join MERGEJOIN[82][tables = [p, t, d, s, p2]] in Stage 'Reducer 5' is a cross product
PREHOOK: query: select * from COLUMN_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME limit 10
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@column_privileges
-PREHOOK: Input: sys@columns_v2
PREHOOK: Input: sys@dbs
PREHOOK: Input: sys@sds
PREHOOK: Input: sys@tbl_col_privs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
#### A masked pattern was here ####
POSTHOOK: query: select * from COLUMN_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME limit 10
POSTHOOK: type: QUERY
POSTHOOK: Input: information_schema@column_privileges
-POSTHOOK: Input: sys@columns_v2
POSTHOOK: Input: sys@dbs
POSTHOOK: Input: sys@sds
POSTHOOK: Input: sys@tbl_col_privs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
#### A masked pattern was here ####
+Warning: Shuffle Join MERGEJOIN[38][tables = [d, t, p]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: select TABLE_SCHEMA, TABLE_NAME from views order by TABLE_SCHEMA, TABLE_NAME
PREHOOK: type: QUERY
PREHOOK: Input: information_schema@views
PREHOOK: Input: sys@dbs
+PREHOOK: Input: sys@tbl_privs
PREHOOK: Input: sys@tbls
#### A masked pattern was here ####
POSTHOOK: query: select TABLE_SCHEMA, TABLE_NAME from views order by TABLE_SCHEMA, TABLE_NAME
POSTHOOK: type: QUERY
POSTHOOK: Input: information_schema@views
POSTHOOK: Input: sys@dbs
+POSTHOOK: Input: sys@tbl_privs
POSTHOOK: Input: sys@tbls
#### A masked pattern was here ####
information_schema column_privileges