You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kl...@apache.org on 2022/06/14 07:18:01 UTC
[hive] branch master updated: HIVE-26280: Copy more data into COMPLETED_COMPACTIONS for better supportability (Karen Coppage, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
klcopp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 37e61a72990 HIVE-26280: Copy more data into COMPLETED_COMPACTIONS for better supportability (Karen Coppage, reviewed by Denys Kuzmenko)
37e61a72990 is described below
commit 37e61a7299018027ac7d811a183365a8b509f242
Author: Karen Coppage <kl...@apache.org>
AuthorDate: Tue Jun 14 09:17:53 2022 +0200
HIVE-26280: Copy more data into COMPLETED_COMPACTIONS for better supportability (Karen Coppage, reviewed by Denys Kuzmenko)
Closes #3340.
---
.../hive/hive-schema-4.0.0-alpha-2.hive.sql | 33 +++-
...upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql | 194 +++++++++++++++++++++
.../test/results/clientpositive/llap/sysdb.q.out | 15 +-
.../hadoop/hive/metastore/txn/CompactionInfo.java | 9 +
.../hive/metastore/txn/CompactionTxnHandler.java | 17 +-
.../sql/derby/hive-schema-4.0.0-alpha-2.derby.sql | 3 +
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql | 5 +
.../sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql | 3 +
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql | 5 +
.../sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql | 3 +
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql | 5 +
.../oracle/hive-schema-4.0.0-alpha-2.oracle.sql | 3 +
...grade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql | 5 +
.../hive-schema-4.0.0-alpha-2.postgres.sql | 3 +
...ade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql | 4 +
.../upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql | 5 +
16 files changed, 301 insertions(+), 11 deletions(-)
diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql
index 61cafd8ca9e..f6dd0af61a0 100644
--- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql
+++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql
@@ -1090,6 +1090,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `COMPACTION_QUEUE` (
`CQ_HIGHEST_WRITE_ID` bigint,
`CQ_HADOOP_JOB_ID` string,
`CQ_ERROR_MESSAGE` string,
+ `CQ_NEXT_TXN_ID` bigint,
+ `CQ_TXN_ID` bigint,
+ `CQ_COMMIT_TIME` bigint,
`CQ_INITIATOR_ID` string,
`CQ_INITIATOR_VERSION` string,
`CQ_WORKER_VERSION` string,
@@ -1114,6 +1117,9 @@ TBLPROPERTIES (
\"COMPACTION_QUEUE\".\"CQ_HIGHEST_WRITE_ID\",
\"COMPACTION_QUEUE\".\"CQ_HADOOP_JOB_ID\",
\"COMPACTION_QUEUE\".\"CQ_ERROR_MESSAGE\",
+ \"COMPACTION_QUEUE\".\"CQ_NEXT_TXN_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_TXN_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_COMMIT_TIME\",
\"COMPACTION_QUEUE\".\"CQ_INITIATOR_ID\",
\"COMPACTION_QUEUE\".\"CQ_INITIATOR_VERSION\",
\"COMPACTION_QUEUE\".\"CQ_WORKER_VERSION\",
@@ -1138,6 +1144,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `COMPLETED_COMPACTIONS` (
`CC_HIGHEST_WRITE_ID` bigint,
`CC_HADOOP_JOB_ID` string,
`CC_ERROR_MESSAGE` string,
+ `CC_NEXT_TXN_ID` bigint,
+ `CC_TXN_ID` bigint,
+ `CC_COMMIT_TIME` bigint,
`CC_INITIATOR_ID` string,
`CC_INITIATOR_VERSION` string,
`CC_WORKER_VERSION` string
@@ -1162,6 +1171,9 @@ TBLPROPERTIES (
\"COMPLETED_COMPACTIONS\".\"CC_HIGHEST_WRITE_ID\",
\"COMPLETED_COMPACTIONS\".\"CC_HADOOP_JOB_ID\",
\"COMPLETED_COMPACTIONS\".\"CC_ERROR_MESSAGE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_NEXT_TXN_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_TXN_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_COMMIT_TIME\",
\"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_ID\",
\"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_VERSION\",
\"COMPLETED_COMPACTIONS\".\"CC_WORKER_VERSION\"
@@ -1186,11 +1198,16 @@ CREATE OR REPLACE VIEW `COMPACTIONS`
`C_DURATION`,
`C_HADOOP_JOB_ID`,
`C_RUN_AS`,
+ `C_ERROR_MESSAGE`,
+ `C_NEXT_TXN_ID`,
+ `C_TXN_ID`,
+ `C_COMMIT_TIME`,
`C_HIGHEST_WRITE_ID`,
`C_INITIATOR_HOST`,
`C_INITIATOR_ID`,
`C_INITIATOR_VERSION`,
- `C_CLEANER_START`
+ `C_CLEANER_START`,
+ `C_TBLPROPERTIES`
) AS
SELECT
CC_ID,
@@ -1209,11 +1226,16 @@ SELECT
CASE WHEN CC_END IS NULL THEN cast (null as string) ELSE CC_END-CC_START END,
CC_HADOOP_JOB_ID,
CC_RUN_AS,
+ CC_ERROR_MESSAGE,
+ CC_NEXT_TXN_ID,
+ CC_TXN_ID,
+ CC_COMMIT_TIME,
CC_HIGHEST_WRITE_ID,
CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[0] END,
CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[size(split(CC_INITIATOR_ID,"-"))-1] END,
CC_INITIATOR_VERSION,
- NULL
+ NULL,
+ CC_TBLPROPERTIES
FROM COMPLETED_COMPACTIONS
UNION ALL
SELECT
@@ -1232,11 +1254,16 @@ SELECT
cast (null as string),
CQ_HADOOP_JOB_ID,
CQ_RUN_AS,
+ CQ_ERROR_MESSAGE,
+ CQ_NEXT_TXN_ID,
+ CQ_TXN_ID,
+ CQ_COMMIT_TIME,
CQ_HIGHEST_WRITE_ID,
CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[0] END,
CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[size(split(CQ_INITIATOR_ID,"-"))-1] END,
CQ_INITIATOR_VERSION,
- CQ_CLEANER_START
+ CQ_CLEANER_START,
+ CQ_TBLPROPERTIES
FROM COMPACTION_QUEUE;
CREATE EXTERNAL TABLE IF NOT EXISTS `SCHEDULED_QUERIES` (
diff --git a/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql b/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql
index a81d6881edc..73d9a5234d7 100644
--- a/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql
+++ b/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql
@@ -16,4 +16,198 @@ TBLPROPERTIES (
FROM \"MIN_HISTORY_LEVEL\""
);
+-- HIVE-26280
+DROP TABLE IF EXISTS `COMPACTION_QUEUE`;
+CREATE EXTERNAL TABLE IF NOT EXISTS `COMPACTION_QUEUE` (
+ `CQ_ID` bigint,
+ `CQ_DATABASE` string,
+ `CQ_TABLE` string,
+ `CQ_PARTITION` string,
+ `CQ_STATE` string,
+ `CQ_TYPE` string,
+ `CQ_TBLPROPERTIES` string,
+ `CQ_WORKER_ID` string,
+ `CQ_ENQUEUE_TIME` bigint,
+ `CQ_START` bigint,
+ `CQ_RUN_AS` string,
+ `CQ_HIGHEST_WRITE_ID` bigint,
+ `CQ_HADOOP_JOB_ID` string,
+ `CQ_ERROR_MESSAGE` string,
+ `CQ_NEXT_TXN_ID` bigint,
+ `CQ_TXN_ID` bigint,
+ `CQ_COMMIT_TIME` bigint,
+ `CQ_INITIATOR_ID` string,
+ `CQ_INITIATOR_VERSION` string,
+ `CQ_WORKER_VERSION` string,
+ `CQ_CLEANER_START` bigint
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+"hive.sql.database.type" = "METASTORE",
+"hive.sql.query" =
+"SELECT
+ \"COMPACTION_QUEUE\".\"CQ_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_DATABASE\",
+ \"COMPACTION_QUEUE\".\"CQ_TABLE\",
+ \"COMPACTION_QUEUE\".\"CQ_PARTITION\",
+ \"COMPACTION_QUEUE\".\"CQ_STATE\",
+ \"COMPACTION_QUEUE\".\"CQ_TYPE\",
+ \"COMPACTION_QUEUE\".\"CQ_TBLPROPERTIES\",
+ \"COMPACTION_QUEUE\".\"CQ_WORKER_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_ENQUEUE_TIME\",
+ \"COMPACTION_QUEUE\".\"CQ_START\",
+ \"COMPACTION_QUEUE\".\"CQ_RUN_AS\",
+ \"COMPACTION_QUEUE\".\"CQ_HIGHEST_WRITE_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_HADOOP_JOB_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_ERROR_MESSAGE\",
+ \"COMPACTION_QUEUE\".\"CQ_NEXT_TXN_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_TXN_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_COMMIT_TIME\",
+ \"COMPACTION_QUEUE\".\"CQ_INITIATOR_ID\",
+ \"COMPACTION_QUEUE\".\"CQ_INITIATOR_VERSION\",
+ \"COMPACTION_QUEUE\".\"CQ_WORKER_VERSION\",
+ \"COMPACTION_QUEUE\".\"CQ_CLEANER_START\"
+FROM \"COMPACTION_QUEUE\"
+"
+);
+
+DROP TABLE IF EXISTS `COMPLETED_COMPACTIONS`;
+CREATE EXTERNAL TABLE IF NOT EXISTS `COMPLETED_COMPACTIONS` (
+ `CC_ID` bigint,
+ `CC_DATABASE` string,
+ `CC_TABLE` string,
+ `CC_PARTITION` string,
+ `CC_STATE` string,
+ `CC_TYPE` string,
+ `CC_TBLPROPERTIES` string,
+ `CC_WORKER_ID` string,
+ `CC_ENQUEUE_TIME` bigint,
+ `CC_START` bigint,
+ `CC_END` bigint,
+ `CC_RUN_AS` string,
+ `CC_HIGHEST_WRITE_ID` bigint,
+ `CC_HADOOP_JOB_ID` string,
+ `CC_ERROR_MESSAGE` string,
+ `CC_NEXT_TXN_ID` bigint,
+ `CC_TXN_ID` bigint,
+ `CC_COMMIT_TIME` bigint,
+ `CC_INITIATOR_ID` string,
+ `CC_INITIATOR_VERSION` string,
+ `CC_WORKER_VERSION` string
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+"hive.sql.database.type" = "METASTORE",
+"hive.sql.query" =
+"SELECT
+ \"COMPLETED_COMPACTIONS\".\"CC_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_DATABASE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_TABLE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_PARTITION\",
+ \"COMPLETED_COMPACTIONS\".\"CC_STATE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_TYPE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_TBLPROPERTIES\",
+ \"COMPLETED_COMPACTIONS\".\"CC_WORKER_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_ENQUEUE_TIME\",
+ \"COMPLETED_COMPACTIONS\".\"CC_START\",
+ \"COMPLETED_COMPACTIONS\".\"CC_END\",
+ \"COMPLETED_COMPACTIONS\".\"CC_RUN_AS\",
+ \"COMPLETED_COMPACTIONS\".\"CC_HIGHEST_WRITE_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_HADOOP_JOB_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_ERROR_MESSAGE\",
+ \"COMPLETED_COMPACTIONS\".\"CC_NEXT_TXN_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_TXN_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_COMMIT_TIME\",
+ \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_ID\",
+ \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_VERSION\",
+ \"COMPLETED_COMPACTIONS\".\"CC_WORKER_VERSION\"
+FROM \"COMPLETED_COMPACTIONS\"
+"
+);
+
+CREATE OR REPLACE VIEW `COMPACTIONS`
+(
+ `C_ID`,
+ `C_CATALOG`,
+ `C_DATABASE`,
+ `C_TABLE`,
+ `C_PARTITION`,
+ `C_TYPE`,
+ `C_STATE`,
+ `C_WORKER_HOST`,
+ `C_WORKER_ID`,
+ `C_WORKER_VERSION`,
+ `C_ENQUEUE_TIME`,
+ `C_START`,
+ `C_DURATION`,
+ `C_HADOOP_JOB_ID`,
+ `C_RUN_AS`,
+ `C_ERROR_MESSAGE`,
+ `C_NEXT_TXN_ID`,
+ `C_TXN_ID`,
+ `C_COMMIT_TIME`,
+ `C_HIGHEST_WRITE_ID`,
+ `C_INITIATOR_HOST`,
+ `C_INITIATOR_ID`,
+ `C_INITIATOR_VERSION`,
+ `C_CLEANER_START`,
+ `C_TBLPROPERTIES`
+) AS
+SELECT
+ CC_ID,
+ 'default',
+ CC_DATABASE,
+ CC_TABLE,
+ CC_PARTITION,
+ CASE WHEN CC_TYPE = 'i' THEN 'minor' WHEN CC_TYPE = 'a' THEN 'major' ELSE 'UNKNOWN' END,
+ CASE WHEN CC_STATE = 'f' THEN 'failed' WHEN CC_STATE = 's' THEN 'succeeded'
+ WHEN CC_STATE = 'a' THEN 'did not initiate' WHEN CC_STATE = 'c' THEN 'refused' ELSE 'UNKNOWN' END,
+ CASE WHEN CC_WORKER_ID IS NULL THEN cast (null as string) ELSE split(CC_WORKER_ID,"-")[0] END,
+ CASE WHEN CC_WORKER_ID IS NULL THEN cast (null as string) ELSE split(CC_WORKER_ID,"-")[size(split(CC_WORKER_ID,"-"))-1] END,
+ CC_WORKER_VERSION,
+ CC_ENQUEUE_TIME,
+ CC_START,
+ CASE WHEN CC_END IS NULL THEN cast (null as string) ELSE CC_END-CC_START END,
+ CC_HADOOP_JOB_ID,
+ CC_RUN_AS,
+ CC_ERROR_MESSAGE,
+ CC_NEXT_TXN_ID,
+ CC_TXN_ID,
+ CC_COMMIT_TIME,
+ CC_HIGHEST_WRITE_ID,
+ CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[0] END,
+ CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[size(split(CC_INITIATOR_ID,"-"))-1] END,
+ CC_INITIATOR_VERSION,
+ NULL,
+ CC_TBLPROPERTIES
+FROM COMPLETED_COMPACTIONS
+UNION ALL
+SELECT
+ CQ_ID,
+ 'default',
+ CQ_DATABASE,
+ CQ_TABLE,
+ CQ_PARTITION,
+ CASE WHEN CQ_TYPE = 'i' THEN 'minor' WHEN CQ_TYPE = 'a' THEN 'major' ELSE 'UNKNOWN' END,
+ CASE WHEN CQ_STATE = 'i' THEN 'initiated' WHEN CQ_STATE = 'w' THEN 'working' WHEN CQ_STATE = 'r' THEN 'ready for cleaning' ELSE 'UNKNOWN' END,
+ CASE WHEN CQ_WORKER_ID IS NULL THEN NULL ELSE split(CQ_WORKER_ID,"-")[0] END,
+ CASE WHEN CQ_WORKER_ID IS NULL THEN NULL ELSE split(CQ_WORKER_ID,"-")[size(split(CQ_WORKER_ID,"-"))-1] END,
+ CQ_WORKER_VERSION,
+ CQ_ENQUEUE_TIME,
+ CQ_START,
+ cast (null as string),
+ CQ_HADOOP_JOB_ID,
+ CQ_RUN_AS,
+ CQ_ERROR_MESSAGE,
+ CQ_NEXT_TXN_ID,
+ CQ_TXN_ID,
+ CQ_COMMIT_TIME,
+ CQ_HIGHEST_WRITE_ID,
+ CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[0] END,
+ CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[size(split(CQ_INITIATOR_ID,"-"))-1] END,
+ CQ_INITIATOR_VERSION,
+ CQ_CLEANER_START,
+ CQ_TBLPROPERTIES
+FROM COMPACTION_QUEUE;
+
SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2';
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index b5c0869bba6..d2bf198a4ad 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -470,6 +470,7 @@ columns_v2 comment
columns_v2 integer_idx
columns_v2 type_name
compaction_queue cq_cleaner_start
+compaction_queue cq_commit_time
compaction_queue cq_database
compaction_queue cq_enqueue_time
compaction_queue cq_error_message
@@ -478,12 +479,14 @@ compaction_queue cq_highest_write_id
compaction_queue cq_id
compaction_queue cq_initiator_id
compaction_queue cq_initiator_version
+compaction_queue cq_next_txn_id
compaction_queue cq_partition
compaction_queue cq_run_as
compaction_queue cq_start
compaction_queue cq_state
compaction_queue cq_table
compaction_queue cq_tblproperties
+compaction_queue cq_txn_id
compaction_queue cq_type
compaction_queue cq_worker_id
compaction_queue cq_worker_version
@@ -491,12 +494,14 @@ compactions c_catalog
compactions c_catalog
compactions c_cleaner_start
compactions c_cleaner_start
+compactions c_commit_time
compactions c_database
compactions c_database
compactions c_duration
compactions c_duration
compactions c_enqueue_time
compactions c_enqueue_time
+compactions c_error_message
#### A masked pattern was here ####
compactions c_highest_write_id
compactions c_highest_write_id
@@ -508,6 +513,7 @@ compactions c_initiator_id
compactions c_initiator_id
compactions c_initiator_version
compactions c_initiator_version
+compactions c_next_txn_id
compactions c_partition
compactions c_partition
compactions c_run_as
@@ -518,6 +524,8 @@ compactions c_state
compactions c_state
compactions c_table
compactions c_table
+compactions c_tblproperties
+compactions c_txn_id
compactions c_type
compactions c_type
compactions c_worker_host
@@ -526,6 +534,7 @@ compactions c_worker_id
compactions c_worker_id
compactions c_worker_version
compactions c_worker_version
+completed_compactions cc_commit_time
completed_compactions cc_database
completed_compactions cc_end
completed_compactions cc_enqueue_time
@@ -535,12 +544,14 @@ completed_compactions cc_highest_write_id
completed_compactions cc_id
completed_compactions cc_initiator_id
completed_compactions cc_initiator_version
+completed_compactions cc_next_txn_id
completed_compactions cc_partition
completed_compactions cc_run_as
completed_compactions cc_start
completed_compactions cc_state
completed_compactions cc_table
completed_compactions cc_tblproperties
+completed_compactions cc_txn_id
completed_compactions cc_type
completed_compactions cc_worker_id
completed_compactions cc_worker_version
@@ -1561,8 +1572,8 @@ POSTHOOK: Input: sys@compaction_queue
POSTHOOK: Input: sys@compactions
POSTHOOK: Input: sys@completed_compactions
#### A masked pattern was here ####
-1 default default scr_txn NULL major initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL
-2 default default scr_txn_2 NULL minor initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL
+1 default default scr_txn NULL major initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL NULL
+2 default default scr_txn_2 NULL minor initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL NULL
PREHOOK: query: use INFORMATION_SCHEMA
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:information_schema
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
index 7ac42380a50..e6fe08b9357 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
@@ -59,6 +59,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
public boolean tooManyAborts = false;
public boolean hasOldAbort = false;
public long retryRetention = 0;
+ public long nextTxnId = 0;
+ public long txnId = 0;
+ public long commitTime = 0;
/**
* The highest write id that the compaction job will pay attention to.
@@ -202,6 +205,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
fullCi.initiatorId = rs.getString(17);
fullCi.initiatorVersion = rs.getString(18);
fullCi.retryRetention = rs.getLong(19);
+ fullCi.nextTxnId = rs.getLong(20);
+ fullCi.txnId = rs.getLong(21);
+ fullCi.commitTime = rs.getLong(22);
return fullCi;
}
static void insertIntoCompletedCompactions(PreparedStatement pStmt, CompactionInfo ci, long endTime) throws SQLException, MetaException {
@@ -224,6 +230,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
pStmt.setString(17, ci.workerVersion);
pStmt.setString(18, ci.initiatorId);
pStmt.setString(19, ci.initiatorVersion);
+ pStmt.setLong(20, ci.nextTxnId);
+ pStmt.setLong(21, ci.txnId);
+ pStmt.setLong(22, ci.commitTime);
}
public static CompactionInfo compactionStructToInfo(CompactionInfoStruct cr) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 5753c7863e9..ac2243530f0 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -64,7 +64,8 @@ class CompactionTxnHandler extends TxnHandler {
+ "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+ "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+ "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
- + "\"CQ_RETRY_RETENTION\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
+ + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\""
+ + " FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY =
"SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " +
"WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?";
@@ -535,12 +536,14 @@ class CompactionTxnHandler extends TxnHandler {
+ "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
+ "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
+ "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
- + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\") "
+ + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
+ + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\") "
+ "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+ quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
+ getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
+ "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
- + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\""
+ + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+ + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\" "
+ "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
pStmt = dbConn.prepareStatement(s);
pStmt.setLong(1, info.id);
@@ -1363,7 +1366,8 @@ class CompactionTxnHandler extends TxnHandler {
+ "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+ "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+ "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
- + "\"CQ_RETRY_RETENTION\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?");
+ + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\" "
+ + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?");
pStmt.setLong(1, ci.id);
rs = pStmt.executeQuery();
if (rs.next()) {
@@ -1405,8 +1409,9 @@ class CompactionTxnHandler extends TxnHandler {
+ "(\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", "
+ "\"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", "
+ "\"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", "
- + "\"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\") "
- + "VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?,?,?,?,?)");
+ + "\"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\","
+ + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\") "
+ + "VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?)");
CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
int updCount = pStmt.executeUpdate();
LOG.debug("Inserted " + updCount + " entries into COMPLETED_COMPACTIONS");
diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql
index 172382a7da7..949c144c86b 100644
--- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql
@@ -655,6 +655,9 @@ CREATE TABLE COMPLETED_COMPACTIONS (
CC_META_INFO varchar(2048) for bit data,
CC_HADOOP_JOB_ID varchar(32),
CC_ERROR_MESSAGE clob,
+ CC_NEXT_TXN_ID bigint,
+ CC_TXN_ID bigint,
+ CC_COMMIT_TIME bigint,
CC_INITIATOR_ID varchar(128),
CC_INITIATOR_VERSION varchar(128),
CC_WORKER_VERSION varchar(128)
diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql
index 22c56389d85..bf17bd33380 100644
--- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql
@@ -1,4 +1,9 @@
-- Upgrade MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2
+-- HIVE-26280
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID bigint;
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID bigint;
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME bigint;
+
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1;
diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql
index 0fd9303b279..b12ab46fc4c 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql
@@ -1073,6 +1073,9 @@ CREATE TABLE COMPLETED_COMPACTIONS (
CC_META_INFO varbinary(2048) NULL,
CC_HADOOP_JOB_ID nvarchar(128) NULL,
CC_ERROR_MESSAGE varchar(max) NULL,
+ CC_NEXT_TXN_ID bigint NULL,
+ CC_TXN_ID bigint NULL,
+ CC_COMMIT_TIME bigint NULL,
CC_INITIATOR_ID nvarchar(128) NULL,
CC_INITIATOR_VERSION nvarchar(128) NULL,
CC_WORKER_VERSION nvarchar(128) NULL,
diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql
index 1b1ff800a86..7dd16b77dfe 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql
@@ -1,5 +1,10 @@
SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE;
+-- HIVE-26280
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID bigint NULL;
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID bigint NULL;
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME bigint NULL;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE;
diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql
index c98cd53ee6a..2bbe7b1445b 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql
@@ -1111,6 +1111,9 @@ CREATE TABLE COMPLETED_COMPACTIONS (
CC_META_INFO varbinary(2048),
CC_HADOOP_JOB_ID varchar(32),
CC_ERROR_MESSAGE mediumtext,
+ CC_NEXT_TXN_ID bigint,
+ CC_TXN_ID bigint,
+ CC_COMMIT_TIME bigint,
CC_INITIATOR_ID varchar(128),
CC_INITIATOR_VERSION varchar(128),
CC_WORKER_VERSION varchar(128)
diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql
index a773532bb7d..09eace89b49 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql
@@ -1,5 +1,10 @@
SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE;
+-- HIVE-26280
+ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_NEXT_TXN_ID` bigint;
+ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_TXN_ID` bigint;
+ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_COMMIT_TIME` bigint;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE;
diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql
index 0d2b8ad5f8e..29d6f0ffddf 100644
--- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql
@@ -1118,6 +1118,9 @@ CREATE TABLE COMPLETED_COMPACTIONS (
CC_META_INFO BLOB,
CC_HADOOP_JOB_ID varchar2(32),
CC_ERROR_MESSAGE CLOB,
+ CC_NEXT_TXN_ID NUMBER(19),
+ CC_TXN_ID NUMBER(19),
+ CC_COMMIT_TIME NUMBER(19),
CC_INITIATOR_ID varchar(128),
CC_INITIATOR_VERSION varchar(128),
CC_WORKER_VERSION varchar(128)
diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql
index 79eadf14809..c0fd8f231d3 100644
--- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql
@@ -1,5 +1,10 @@
SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS Status from dual;
+-- HIVE-26280
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID NUMBER(19);
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID NUMBER(19);
+ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME NUMBER(19);
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS Status from dual;
diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql
index 4be408996ab..b72b8d5ca0f 100644
--- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql
@@ -1827,6 +1827,9 @@ CREATE TABLE "COMPLETED_COMPACTIONS" (
"CC_META_INFO" bytea,
"CC_HADOOP_JOB_ID" varchar(32),
"CC_ERROR_MESSAGE" text,
+ "CC_NEXT_TXN_ID" bigint,
+ "CC_TXN_ID" bigint,
+ "CC_COMMIT_TIME" bigint,
"CC_INITIATOR_ID" varchar(128),
"CC_INITIATOR_VERSION" varchar(128),
"CC_WORKER_VERSION" varchar(128)
diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql
index fa03cd9f295..2dad36782f3 100644
--- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql
@@ -1,5 +1,9 @@
SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2';
+-- HIVE-26280
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_NEXT_TXN_ID" bigint;
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_TXN_ID" bigint;
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_COMMIT_TIME" bigint;
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0-alpha-2', "VERSION_COMMENT"='Hive release version 4.0.0-alpha-2' where "VER_ID"=1;
diff --git a/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql
index be224ab4619..ae712a92f20 100644
--- a/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql
+++ b/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql
@@ -131,6 +131,11 @@ CREATE TABLE "COMPACTION_METRICS_CACHE" (
-- HIVE-25993
ALTER TABLE "COMPACTION_QUEUE" ADD "CQ_RETRY_RETENTION" integer NOT NULL DEFAULT 0;
+-- HIVE-26280
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_NEXT_TXN_ID" bigint;
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_TXN_ID" bigint;
+ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_COMMIT_TIME" bigint;
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0-alpha-2', "VERSION_COMMENT"='Hive release version 4.0.0-alpha-2' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 3.1.3000 to 4.0.0-alpha-2';