You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2020/05/06 23:26:41 UTC
[hive] branch master updated: HIVE-23318: TxnHandler should delete
from MATERIALIZATION_REBUILD_LOCKS on need basis (Marton Bod,
reviwed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 54b8799 HIVE-23318: TxnHandler should delete from MATERIALIZATION_REBUILD_LOCKS on need basis (Marton Bod, reviwed by Jesus Camacho Rodriguez)
54b8799 is described below
commit 54b87999fa0c23f9902faae609e7441e0693a22b
Author: Marton Bod <mb...@cloudera.com>
AuthorDate: Wed May 6 16:24:55 2020 -0700
HIVE-23318: TxnHandler should delete from MATERIALIZATION_REBUILD_LOCKS on need basis (Marton Bod, reviwed by Jesus Camacho Rodriguez)
---
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 24 ++++++++++++++--------
.../hadoop/hive/ql/parse/TestParseUtils.java | 7 ++++++-
.../apache/hadoop/hive/metastore/api/TxnType.java | 5 ++++-
.../src/gen/thrift/gen-php/metastore/Types.php | 2 ++
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 3 +++
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 5 +++--
.../src/main/thrift/hive_metastore.thrift | 3 ++-
.../hadoop/hive/metastore/txn/TxnHandler.java | 4 +++-
8 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 998c05e..66404ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -3152,14 +3152,22 @@ public class AcidUtils {
public static TxnType getTxnType(Configuration conf, ASTNode tree) {
final ASTSearcher astSearcher = new ASTSearcher();
- return (HiveConf.getBoolVar(conf, ConfVars.HIVE_TXN_READONLY_ENABLED) &&
- tree.getToken().getType() == HiveParser.TOK_QUERY &&
- Stream.of(
- new int[]{HiveParser.TOK_INSERT_INTO},
- new int[]{HiveParser.TOK_INSERT, HiveParser.TOK_TAB})
- .noneMatch(pattern ->
- astSearcher.simpleBreadthFirstSearch(tree, pattern) != null)) ?
- TxnType.READ_ONLY : TxnType.DEFAULT;
+ // check if read-only txn
+ if (HiveConf.getBoolVar(conf, ConfVars.HIVE_TXN_READONLY_ENABLED) &&
+ tree.getToken().getType() == HiveParser.TOK_QUERY &&
+ Stream.of(
+ new int[]{HiveParser.TOK_INSERT_INTO},
+ new int[]{HiveParser.TOK_INSERT, HiveParser.TOK_TAB})
+ .noneMatch(pattern -> astSearcher.simpleBreadthFirstSearch(tree, pattern) != null)) {
+ return TxnType.READ_ONLY;
+ }
+
+ // check if txn has a materialized view rebuild
+ if (tree.getToken().getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD) {
+ return TxnType.MATER_VIEW_REBUILD;
+ }
+
+ return TxnType.DEFAULT;
}
public static List<HdfsFileStatusWithId> findBaseFiles(
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java
index e91a7ed..69cb96d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseUtils.java
@@ -89,6 +89,10 @@ public class TestParseUtils {
"WHEN NOT MATCHED THEN " +
" INSERT VALUES (src.col_a, src.col_b)",
TxnType.DEFAULT},
+
+ {"CREATE MATERIALIZED VIEW matview AS SELECT * FROM b", TxnType.DEFAULT},
+ {"ALTER MATERIALIZED VIEW matview REBUILD", TxnType.MATER_VIEW_REBUILD},
+ {"ALTER MATERIALIZED VIEW matview DISABLE REWRITE", TxnType.DEFAULT}
});
}
@@ -101,7 +105,8 @@ public class TestParseUtils {
@Test
public void testTxnTypeWithDisabledReadOnlyFeature() throws ParseException {
enableReadOnlyTxnFeature(false);
- Assert.assertEquals(AcidUtils.getTxnType(conf, ParseUtils.parse(query)), TxnType.DEFAULT);
+ Assert.assertEquals(AcidUtils.getTxnType(conf, ParseUtils.parse(query)),
+ txnType == TxnType.READ_ONLY ? TxnType.DEFAULT : txnType);
}
private void enableReadOnlyTxnFeature(boolean featureFlag) {
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
index 349f37f..7d050c5 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
@@ -15,7 +15,8 @@ public enum TxnType implements org.apache.thrift.TEnum {
DEFAULT(0),
REPL_CREATED(1),
READ_ONLY(2),
- COMPACTION(3);
+ COMPACTION(3),
+ MATER_VIEW_REBUILD(4);
private final int value;
@@ -44,6 +45,8 @@ public enum TxnType implements org.apache.thrift.TEnum {
return READ_ONLY;
case 3:
return COMPACTION;
+ case 4:
+ return MATER_VIEW_REBUILD;
default:
return null;
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
index 302c340..65dd8b3 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
@@ -228,11 +228,13 @@ final class TxnType {
const REPL_CREATED = 1;
const READ_ONLY = 2;
const COMPACTION = 3;
+ const MATER_VIEW_REBUILD = 4;
static public $__names = array(
0 => 'DEFAULT',
1 => 'REPL_CREATED',
2 => 'READ_ONLY',
3 => 'COMPACTION',
+ 4 => 'MATER_VIEW_REBUILD',
);
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 28c971f..7b6b6d2 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -341,12 +341,14 @@ class TxnType:
REPL_CREATED = 1
READ_ONLY = 2
COMPACTION = 3
+ MATER_VIEW_REBUILD = 4
_VALUES_TO_NAMES = {
0: "DEFAULT",
1: "REPL_CREATED",
2: "READ_ONLY",
3: "COMPACTION",
+ 4: "MATER_VIEW_REBUILD",
}
_NAMES_TO_VALUES = {
@@ -354,6 +356,7 @@ class TxnType:
"REPL_CREATED": 1,
"READ_ONLY": 2,
"COMPACTION": 3,
+ "MATER_VIEW_REBUILD": 4,
}
class GetTablesExtRequestFields:
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index cdf97fc..5d47578 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -161,8 +161,9 @@ module TxnType
REPL_CREATED = 1
READ_ONLY = 2
COMPACTION = 3
- VALUE_MAP = {0 => "DEFAULT", 1 => "REPL_CREATED", 2 => "READ_ONLY", 3 => "COMPACTION"}
- VALID_VALUES = Set.new([DEFAULT, REPL_CREATED, READ_ONLY, COMPACTION]).freeze
+ MATER_VIEW_REBUILD = 4
+ VALUE_MAP = {0 => "DEFAULT", 1 => "REPL_CREATED", 2 => "READ_ONLY", 3 => "COMPACTION", 4 => "MATER_VIEW_REBUILD"}
+ VALID_VALUES = Set.new([DEFAULT, REPL_CREATED, READ_ONLY, COMPACTION, MATER_VIEW_REBUILD]).freeze
end
module GetTablesExtRequestFields
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index c78aeb4..42de537 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -898,7 +898,8 @@ enum TxnType {
DEFAULT = 0,
REPL_CREATED = 1,
READ_ONLY = 2,
- COMPACTION = 3
+ COMPACTION = 3,
+ MATER_VIEW_REBUILD = 4
}
// specifies which info to return with GetTablesExtRequest
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 8fded60..7c39375 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -1590,7 +1590,9 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
queryBatch.add("DELETE FROM \"HIVE_LOCKS\" WHERE \"HL_TXNID\" = " + txnid);
// DO NOT remove the transaction from the TXN table, the cleaner will remove it when appropriate
queryBatch.add("UPDATE \"TXNS\" SET \"TXN_STATE\" = " + quoteChar(TXN_COMMITTED) + " WHERE \"TXN_ID\" = " + txnid);
- queryBatch.add("DELETE FROM \"MATERIALIZATION_REBUILD_LOCKS\" WHERE \"MRL_TXN_ID\" = " + txnid);
+ if (txnType == TxnType.MATER_VIEW_REBUILD) {
+ queryBatch.add("DELETE FROM \"MATERIALIZATION_REBUILD_LOCKS\" WHERE \"MRL_TXN_ID\" = " + txnid);
+ }
// execute all in one batch
executeQueriesInBatchNoCount(dbProduct, stmt, queryBatch, maxBatchSize);