You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by an...@apache.org on 2020/06/18 03:49:51 UTC
[hive] branch master updated: HIVE-23585: Retrieve replication
instance metrics details (Aasha Medhi, reviewed by Pravin Kumar Sinha)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9ec1470 HIVE-23585: Retrieve replication instance metrics details (Aasha Medhi, reviewed by Pravin Kumar Sinha)
9ec1470 is described below
commit 9ec1470027e59be322a0bb5166d289cc9412a9ce
Author: Anishek Agarwal <an...@gmail.com>
AuthorDate: Thu Jun 18 09:19:39 2020 +0530
HIVE-23585: Retrieve replication instance metrics details (Aasha Medhi, reviewed by Pravin Kumar Sinha)
---
.../upgrade/hive/hive-schema-4.0.0.hive.sql | 20 ++++++
.../upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql | 21 ++++++
.../clientpositive/replication_metrics_ingest.q | 40 ++++++++++++
.../llap/replication_metrics_ingest.q.out | 76 ++++++++++++++++++++++
.../results/clientpositive/llap/resourceplan.q.out | 4 ++
.../llap/strict_managed_tables_sysdb.q.out | 6 ++
.../test/results/clientpositive/llap/sysdb.q.out | 12 +++-
7 files changed, 178 insertions(+), 1 deletion(-)
diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
index e7fe916..8fe0b17 100644
--- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
@@ -1436,6 +1436,26 @@ SELECT DISTINCT
HL.`HL_BLOCKEDBY_INT_ID`
FROM SYS.`HIVE_LOCKS` AS HL;
+CREATE EXTERNAL TABLE IF NOT EXISTS `REPLICATION_METRICS` (
+ `SCHEDULED_EXECUTION_ID` bigint,
+ `POLICY_NAME` string,
+ `DUMP_EXECUTION_ID` bigint,
+ `METADATA` string,
+ `PROGRESS` string
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+"hive.sql.database.type" = "METASTORE",
+"hive.sql.query" =
+"SELECT
+ \"RM_SCHEDULED_EXECUTION_ID\",
+ \"RM_POLICY\",
+ \"RM_DUMP_EXECUTION_ID\",
+ \"RM_METADATA\",
+ \"RM_PROGRESS\"
+FROM \"REPLICATION_METRICS\""
+);
+
CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA;
USE INFORMATION_SCHEMA;
diff --git a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
index 3fb3f7a..7862f6d 100644
--- a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
@@ -497,6 +497,26 @@ SELECT DISTINCT
HL.`HL_BLOCKEDBY_INT_ID`
FROM SYS.`HIVE_LOCKS` AS HL;
+CREATE EXTERNAL TABLE IF NOT EXISTS `REPLICATION_METRICS` (
+ `SCHEDULED_EXECUTION_ID` bigint,
+ `POLICY_NAME` string,
+ `DUMP_EXECUTION_ID` bigint,
+ `METADATA` string,
+ `PROGRESS` string
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+"hive.sql.database.type" = "METASTORE",
+"hive.sql.query" =
+"SELECT
+ \"RM_SCHEDULED_EXECUTION_ID\",
+ \"RM_POLICY\",
+ \"RM_DUMP_EXECUTION_ID\",
+ \"RM_METADATA\",
+ \"RM_PROGRESS\"
+FROM \"REPLICATION_METRICS\""
+);
+
DROP TABLE IF EXISTS `VERSION`;
CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`,
@@ -775,4 +795,5 @@ WHERE
OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer());
+
SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0';
diff --git a/ql/src/test/queries/clientpositive/replication_metrics_ingest.q b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q
new file mode 100644
index 0000000..7f0ef81
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q
@@ -0,0 +1,40 @@
+--! qt:authorizer
+--! qt:scheduledqueryservice
+--! qt:sysdb
+
+set hive.repl.rootdir=${system:test.tmp.dir}/repl;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl/sentinel;
+dfs -rmr ${system:test.tmp.dir}/repl;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl;
+
+set user.name=hive_admin_user;
+set role ADMIN;
+
+drop database if exists src cascade;
+drop database if exists destination cascade;
+
+create database src with DBPROPERTIES ('repl.source.for' = '1,2,3');
+
+create table src.t(id int, cnt int);
+
+-- add data to table
+insert into src.t values(1,1);
+
+create scheduled query repl1 every 15 minutes as repl dump src
+with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl');
+
+alter scheduled query repl1 execute;
+
+!sleep 50;
+
+create scheduled query repl2 every 15 minutes as repl load src into destination
+with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl');
+
+alter scheduled query repl2 execute;
+
+!sleep 50;
+
+show databases;
+
+select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics;
diff --git a/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out
new file mode 100644
index 0000000..1380ccd
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out
@@ -0,0 +1,76 @@
+#### A masked pattern was here ####
+PREHOOK: query: set role ADMIN
+PREHOOK: type: SHOW_ROLES
+POSTHOOK: query: set role ADMIN
+POSTHOOK: type: SHOW_ROLES
+PREHOOK: query: drop database if exists src cascade
+PREHOOK: type: DROPDATABASE
+POSTHOOK: query: drop database if exists src cascade
+POSTHOOK: type: DROPDATABASE
+PREHOOK: query: drop database if exists destination cascade
+PREHOOK: type: DROPDATABASE
+POSTHOOK: query: drop database if exists destination cascade
+POSTHOOK: type: DROPDATABASE
+PREHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3')
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:src
+POSTHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3')
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:src
+PREHOOK: query: create table src.t(id int, cnt int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:src
+PREHOOK: Output: src@t
+POSTHOOK: query: create table src.t(id int, cnt int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:src
+POSTHOOK: Output: src@t
+PREHOOK: query: insert into src.t values(1,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: src@t
+POSTHOOK: query: insert into src.t values(1,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: src@t
+POSTHOOK: Lineage: t.cnt SCRIPT []
+POSTHOOK: Lineage: t.id SCRIPT []
+PREHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src
+#### A masked pattern was here ####
+PREHOOK: type: CREATE SCHEDULED QUERY
+POSTHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src
+#### A masked pattern was here ####
+POSTHOOK: type: CREATE SCHEDULED QUERY
+PREHOOK: query: alter scheduled query repl1 execute
+PREHOOK: type: ALTER SCHEDULED QUERY
+POSTHOOK: query: alter scheduled query repl1 execute
+POSTHOOK: type: ALTER SCHEDULED QUERY
+PREHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination
+#### A masked pattern was here ####
+PREHOOK: type: CREATE SCHEDULED QUERY
+POSTHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination
+#### A masked pattern was here ####
+POSTHOOK: type: CREATE SCHEDULED QUERY
+PREHOOK: query: alter scheduled query repl2 execute
+PREHOOK: type: ALTER SCHEDULED QUERY
+POSTHOOK: query: alter scheduled query repl2 execute
+POSTHOOK: type: ALTER SCHEDULED QUERY
+PREHOOK: query: show databases
+PREHOOK: type: SHOWDATABASES
+POSTHOOK: query: show databases
+POSTHOOK: type: SHOWDATABASES
+default
+destination
+information_schema
+src
+sys
+PREHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics
+PREHOOK: type: QUERY
+PREHOOK: Input: sys@replication_metrics
+#### A masked pattern was here ####
+POSTHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: sys@replication_metrics
+#### A masked pattern was here ####
+1 repl1 0
+2 repl2 1
diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
index 24190d3..dddf147 100644
--- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
+++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
@@ -202,6 +202,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user
sys partitions hive_test_user USER INSERT true -1 hive_test_user
sys partitions hive_test_user USER SELECT true -1 hive_test_user
sys partitions hive_test_user USER UPDATE true -1 hive_test_user
+sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user
+sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user
+sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user
+sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user
sys role_map hive_test_user USER DELETE true -1 hive_test_user
sys role_map hive_test_user USER INSERT true -1 hive_test_user
sys role_map hive_test_user USER SELECT true -1 hive_test_user
diff --git a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
index 96bdecd..54de5a9 100644
--- a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out
@@ -292,6 +292,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user
sys partitions hive_test_user USER INSERT true -1 hive_test_user
sys partitions hive_test_user USER SELECT true -1 hive_test_user
sys partitions hive_test_user USER UPDATE true -1 hive_test_user
+sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user
+sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user
+sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user
+sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user
sys role_map hive_test_user USER DELETE true -1 hive_test_user
sys role_map hive_test_user USER INSERT true -1 hive_test_user
sys role_map hive_test_user USER SELECT true -1 hive_test_user
@@ -480,6 +484,7 @@ PREHOOK: Output: sys@partition_keys
PREHOOK: Output: sys@partition_params
PREHOOK: Output: sys@partition_stats_view
PREHOOK: Output: sys@partitions
+PREHOOK: Output: sys@replication_metrics
PREHOOK: Output: sys@role_map
PREHOOK: Output: sys@roles
PREHOOK: Output: sys@scheduled_executions
@@ -539,6 +544,7 @@ POSTHOOK: Output: sys@partition_keys
POSTHOOK: Output: sys@partition_params
POSTHOOK: Output: sys@partition_stats_view
POSTHOOK: Output: sys@partitions
+POSTHOOK: Output: sys@replication_metrics
POSTHOOK: Output: sys@role_map
POSTHOOK: Output: sys@roles
POSTHOOK: Output: sys@scheduled_executions
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index 26e3200..c78fc24 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -240,6 +240,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user
sys partitions hive_test_user USER INSERT true -1 hive_test_user
sys partitions hive_test_user USER SELECT true -1 hive_test_user
sys partitions hive_test_user USER UPDATE true -1 hive_test_user
+sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user
+sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user
+sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user
+sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user
sys role_map hive_test_user USER DELETE true -1 hive_test_user
sys role_map hive_test_user USER INSERT true -1 hive_test_user
sys role_map hive_test_user USER SELECT true -1 hive_test_user
@@ -680,6 +684,11 @@ partitions part_id
partitions part_name
partitions sd_id
partitions tbl_id
+replication_metrics dump_execution_id
+replication_metrics metadata
+replication_metrics policy_name
+replication_metrics progress
+replication_metrics scheduled_execution_id
role_map add_time
role_map grant_option
role_map grantor
@@ -1179,7 +1188,7 @@ POSTHOOK: query: select count(*) from sds
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@sds
#### A masked pattern was here ####
-72
+73
PREHOOK: query: select param_key, param_value from sd_params order by param_key, param_value limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@sd_params
@@ -1585,6 +1594,7 @@ default sys partition_keys BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys partition_params BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys partition_stats_view VIEW NULL NULL NULL NULL NULL NO NO NULL
default sys partitions BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
+default sys replication_metrics BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys role_map BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys roles BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL
default sys scheduled_executions BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL