You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/05/27 06:47:29 UTC
hive git commit: HIVE-19340: Disable timeout of transactions opened
by replication task at target cluster (Mahesh Kumar Behera,
reviewed by Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master 1f92316d0 -> 702a67657
HIVE-19340: Disable timeout of transactions opened by replication task at target cluster (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/702a6765
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/702a6765
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/702a6765
Branch: refs/heads/master
Commit: 702a676570efee20915ce6832fe7ddee42611ce3
Parents: 1f92316
Author: Sankar Hariappan <sa...@apache.org>
Authored: Sun May 27 12:16:53 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Sun May 27 12:16:53 2018 +0530
----------------------------------------------------------------------
.../hive/metastore/txn/TestTxnHandler.java | 31 +++++++++++++++++++-
.../hadoop/hive/metastore/txn/TxnDbUtil.java | 1 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 21 +++++++++++--
.../main/sql/derby/hive-schema-3.0.0.derby.sql | 3 +-
.../sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 3 ++
.../main/sql/mssql/hive-schema-3.0.0.mssql.sql | 1 +
.../sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 3 ++
.../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 3 +-
.../sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 5 +++-
.../sql/oracle/hive-schema-3.0.0.oracle.sql | 3 +-
.../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 5 +++-
.../sql/postgres/hive-schema-3.0.0.postgres.sql | 3 +-
.../upgrade-2.3.0-to-3.0.0.postgres.sql | 3 ++
13 files changed, 75 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 372c709..9294c2b 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -1169,8 +1169,37 @@ public class TestTxnHandler {
} finally {
txnHandler.setTimeout(timeout);
}
+ }
-
+ @Test
+ public void testReplTimeouts() throws Exception {
+ long timeout = txnHandler.setTimeout(1);
+ try {
+ OpenTxnRequest request = new OpenTxnRequest(3, "me", "localhost");
+ OpenTxnsResponse response = txnHandler.openTxns(request);
+ request.setReplPolicy("default.*");
+ request.setReplSrcTxnIds(response.getTxn_ids());
+ OpenTxnsResponse responseRepl = txnHandler.openTxns(request);
+ Thread.sleep(10);
+ txnHandler.performTimeOuts();
+ GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo();
+ int numAborted = 0;
+ int numOpen = 0;
+ for (TxnInfo txnInfo : rsp.getOpen_txns()) {
+ if (TxnState.ABORTED == txnInfo.getState()) {
+ assertTrue(response.getTxn_ids().contains(txnInfo.getId()));
+ numAborted++;
+ }
+ if (TxnState.OPEN == txnInfo.getState()) {
+ assertTrue(responseRepl.getTxn_ids().contains(txnInfo.getId()));
+ numOpen++;
+ }
+ }
+ assertEquals(3, numAborted);
+ assertEquals(3, numOpen);
+ } finally {
+ txnHandler.setTimeout(timeout);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
index 4597166..7b58cbe 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
@@ -79,6 +79,7 @@ public final class TxnDbUtil {
" TXN_STARTED bigint NOT NULL," +
" TXN_LAST_HEARTBEAT bigint NOT NULL," +
" TXN_USER varchar(128) NOT NULL," +
+ " TXN_TYPE integer," +
" TXN_HOST varchar(128) NOT NULL)");
stmt.execute("CREATE TABLE TXN_COMPONENTS (" +
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index a2b5897..469e699 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -215,6 +215,19 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
//todo: make these like OperationType and remove above char constatns
enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN}
+ public enum TxnType {
+ DEFAULT(0), REPL_CREATED(1), READ_ONLY(2);
+
+ private final int value;
+ TxnType(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+ }
+
// Lock states
static final protected char LOCK_ACQUIRED = 'a';
static final protected char LOCK_WAITING = 'w';
@@ -608,6 +621,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
throws SQLException, MetaException {
int numTxns = rqst.getNum_txns();
ResultSet rs = null;
+ TxnType txnType = TxnType.DEFAULT;
try {
if (rqst.isSetReplPolicy()) {
List<Long> targetTxnIdList = getTargetTxnIdList(rqst.getReplPolicy(), rqst.getReplSrcTxnIds(), stmt);
@@ -621,6 +635,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
rqst.getReplPolicy() + " and Source transaction id : " + rqst.getReplSrcTxnIds().toString());
return targetTxnIdList;
}
+ txnType = TxnType.REPL_CREATED;
}
String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID");
@@ -642,10 +657,10 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
for (long i = first; i < first + numTxns; i++) {
txnIds.add(i);
rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + ","
- + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()));
+ + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()) + "," + txnType.getValue());
}
List<String> queries = sqlGenerator.createInsertValuesStmt(
- "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows);
+ "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host, txn_type)", rows);
for (String q : queries) {
LOG.debug("Going to execute update <" + q + ">");
stmt.execute(q);
@@ -4230,7 +4245,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
while(true) {
stmt = dbConn.createStatement();
String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN +
- "' and txn_last_heartbeat < " + (now - timeout);
+ "' and txn_last_heartbeat < " + (now - timeout) + " and txn_type != " + TxnType.REPL_CREATED.getValue();
//safety valve for extreme cases
s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
LOG.debug("Going to execute query <" + s + ">");
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
index e818e1b..158e51e 100644
--- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
@@ -503,7 +503,8 @@ CREATE TABLE TXNS (
TXN_HOST varchar(128) NOT NULL,
TXN_AGENT_INFO varchar(128),
TXN_META_INFO varchar(128),
- TXN_HEARTBEAT_COUNT integer
+ TXN_HEARTBEAT_COUNT integer,
+ TXN_TYPE integer
);
CREATE TABLE TXN_COMPONENTS (
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql
index 7b7a8a2..ff44d44 100644
--- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql
@@ -279,6 +279,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID)
UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID;
UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
+-- HIVE-19340
+ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer;
+
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
index c88fb18..926f305 100644
--- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
@@ -1084,6 +1084,7 @@ CREATE TABLE TXNS(
TXN_AGENT_INFO nvarchar(128) NULL,
TXN_META_INFO nvarchar(128) NULL,
TXN_HEARTBEAT_COUNT int NULL,
+ TXN_TYPE int NULL,
PRIMARY KEY CLUSTERED
(
TXN_ID ASC
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql
index 14c3deb..12ecc7b 100644
--- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql
@@ -346,6 +346,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID)
UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID;
UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
+-- HIVE-19340
+ALTER TABLE TXNS ADD TXN_TYPE int;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
index c54df55..7f74d61 100644
--- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
@@ -977,7 +977,8 @@ CREATE TABLE TXNS (
TXN_HOST varchar(128) NOT NULL,
TXN_AGENT_INFO varchar(128),
TXN_META_INFO varchar(128),
- TXN_HEARTBEAT_COUNT int
+ TXN_HEARTBEAT_COUNT int,
+ TXN_TYPE int
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE TXN_COMPONENTS (
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
index 9b87563..b556f5c 100644
--- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
@@ -319,8 +319,11 @@ UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL;
+-- HIVE-19340
+ALTER TABLE TXNS ADD COLUMN TXN_TYPE int;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' ';
-ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
\ No newline at end of file
+ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
index 63cc1f7..e5acd2d 100644
--- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
@@ -954,7 +954,8 @@ CREATE TABLE TXNS (
TXN_HOST varchar(128) NOT NULL,
TXN_AGENT_INFO varchar2(128),
TXN_META_INFO varchar2(128),
- TXN_HEARTBEAT_COUNT number(10)
+ TXN_HEARTBEAT_COUNT number(10),
+ TXN_TYPE number(10)
) ROWDEPENDENCIES;
CREATE TABLE TXN_COMPONENTS (
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
index ce3437f..64cb741 100644
--- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
@@ -335,8 +335,11 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID)
UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID;
UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
+-- HIVE-19340
+ALTER TABLE TXNS ADD TXN_TYPE NUMBER(10);
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual;
-ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL;
\ No newline at end of file
+ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
index d210a55..2c94f37 100644
--- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
@@ -1641,7 +1641,8 @@ CREATE TABLE TXNS (
TXN_HOST varchar(128) NOT NULL,
TXN_AGENT_INFO varchar(128),
TXN_META_INFO varchar(128),
- TXN_HEARTBEAT_COUNT integer
+ TXN_HEARTBEAT_COUNT integer,
+ TXN_TYPE integer
);
CREATE TABLE TXN_COMPONENTS (
http://git-wip-us.apache.org/repos/asf/hive/blob/702a6765/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql
index f2571d8..e53db10 100644
--- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql
@@ -354,6 +354,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID)
UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID;
UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
+-- HIVE-19340
+ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer;
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0';