You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2019/01/10 23:52:28 UTC
hive git commit: HIVE-21036 extend OpenTxnRequest with transaction
type (Igor Kryvenko via Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master dcc895016 -> f713140ba
HIVE-21036 extend OpenTxnRequest with transaction type (Igor Kryvenko via Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f713140b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f713140b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f713140b
Branch: refs/heads/master
Commit: f713140ba60d50017d2b97b1f5c5aea7217512dd
Parents: dcc8950
Author: Igor Kryvenko <kr...@gmail.com>
Authored: Thu Jan 10 15:50:41 2019 -0800
Committer: Eugene Koifman <ek...@apache.org>
Committed: Thu Jan 10 15:50:41 2019 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/txn/compactor/Worker.java | 3 +-
.../hive/metastore/api/OpenTxnRequest.java | 129 ++++++++++++++++++-
.../hadoop/hive/metastore/api/TxnType.java | 51 ++++++++
.../src/gen/thrift/gen-php/metastore/Types.php | 36 ++++++
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 35 ++++-
.../gen/thrift/gen-rb/hive_metastore_types.rb | 16 ++-
.../hive/metastore/HiveMetaStoreClient.java | 17 ++-
.../hadoop/hive/metastore/IMetaStoreClient.java | 10 ++
.../src/main/thrift/hive_metastore.thrift | 8 ++
.../hadoop/hive/metastore/txn/TxnHandler.java | 13 --
.../HiveMetaStoreClientPreCatalog.java | 15 ++-
.../hive/metastore/TestHiveMetaStoreTxns.java | 9 ++
12 files changed, 315 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index b645870..e3fab69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnType;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.mapred.JobConf;
import org.apache.thrift.TException;
@@ -155,7 +156,7 @@ public class Worker extends RemoteCompactorThread implements MetaStoreThread {
if (ci.runAs == null) {
ci.runAs = findUserToRunAs(sd.getLocation(), t);
}
- long compactorTxnId = msc.openTxns(ci.runAs, 1).getTxn_ids().get(0);
+ long compactorTxnId = msc.openTxn(ci.runAs, TxnType.COMPACTION);
heartbeater = new CompactionHeartbeater(compactorTxnId, fullTableName, conf);
heartbeater.start();
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
index 19b2c01..b5efffd 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
@@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField AGENT_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("agentInfo", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField REPL_SRC_TXN_IDS_FIELD_DESC = new org.apache.thrift.protocol.TField("replSrcTxnIds", org.apache.thrift.protocol.TType.LIST, (short)6);
+ private static final org.apache.thrift.protocol.TField TXN_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("txn_type", org.apache.thrift.protocol.TType.I32, (short)7);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -57,6 +58,7 @@ import org.slf4j.LoggerFactory;
private String agentInfo; // optional
private String replPolicy; // optional
private List<Long> replSrcTxnIds; // optional
+ private TxnType txn_type; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -65,7 +67,12 @@ import org.slf4j.LoggerFactory;
HOSTNAME((short)3, "hostname"),
AGENT_INFO((short)4, "agentInfo"),
REPL_POLICY((short)5, "replPolicy"),
- REPL_SRC_TXN_IDS((short)6, "replSrcTxnIds");
+ REPL_SRC_TXN_IDS((short)6, "replSrcTxnIds"),
+ /**
+ *
+ * @see TxnType
+ */
+ TXN_TYPE((short)7, "txn_type");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -92,6 +99,8 @@ import org.slf4j.LoggerFactory;
return REPL_POLICY;
case 6: // REPL_SRC_TXN_IDS
return REPL_SRC_TXN_IDS;
+ case 7: // TXN_TYPE
+ return TXN_TYPE;
default:
return null;
}
@@ -134,7 +143,7 @@ import org.slf4j.LoggerFactory;
// isset id assignments
private static final int __NUM_TXNS_ISSET_ID = 0;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.AGENT_INFO,_Fields.REPL_POLICY,_Fields.REPL_SRC_TXN_IDS};
+ private static final _Fields optionals[] = {_Fields.AGENT_INFO,_Fields.REPL_POLICY,_Fields.REPL_SRC_TXN_IDS,_Fields.TXN_TYPE};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -151,6 +160,8 @@ import org.slf4j.LoggerFactory;
tmpMap.put(_Fields.REPL_SRC_TXN_IDS, new org.apache.thrift.meta_data.FieldMetaData("replSrcTxnIds", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
+ tmpMap.put(_Fields.TXN_TYPE, new org.apache.thrift.meta_data.FieldMetaData("txn_type", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TxnType.class)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenTxnRequest.class, metaDataMap);
}
@@ -158,6 +169,8 @@ import org.slf4j.LoggerFactory;
public OpenTxnRequest() {
this.agentInfo = "Unknown";
+ this.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.DEFAULT;
+
}
public OpenTxnRequest(
@@ -194,6 +207,9 @@ import org.slf4j.LoggerFactory;
List<Long> __this__replSrcTxnIds = new ArrayList<Long>(other.replSrcTxnIds);
this.replSrcTxnIds = __this__replSrcTxnIds;
}
+ if (other.isSetTxn_type()) {
+ this.txn_type = other.txn_type;
+ }
}
public OpenTxnRequest deepCopy() {
@@ -210,6 +226,8 @@ import org.slf4j.LoggerFactory;
this.replPolicy = null;
this.replSrcTxnIds = null;
+ this.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.DEFAULT;
+
}
public int getNum_txns() {
@@ -364,6 +382,37 @@ import org.slf4j.LoggerFactory;
}
}
+ /**
+ *
+ * @see TxnType
+ */
+ public TxnType getTxn_type() {
+ return this.txn_type;
+ }
+
+ /**
+ *
+ * @see TxnType
+ */
+ public void setTxn_type(TxnType txn_type) {
+ this.txn_type = txn_type;
+ }
+
+ public void unsetTxn_type() {
+ this.txn_type = null;
+ }
+
+ /** Returns true if field txn_type is set (has been assigned a value) and false otherwise */
+ public boolean isSetTxn_type() {
+ return this.txn_type != null;
+ }
+
+ public void setTxn_typeIsSet(boolean value) {
+ if (!value) {
+ this.txn_type = null;
+ }
+ }
+
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case NUM_TXNS:
@@ -414,6 +463,14 @@ import org.slf4j.LoggerFactory;
}
break;
+ case TXN_TYPE:
+ if (value == null) {
+ unsetTxn_type();
+ } else {
+ setTxn_type((TxnType)value);
+ }
+ break;
+
}
}
@@ -437,6 +494,9 @@ import org.slf4j.LoggerFactory;
case REPL_SRC_TXN_IDS:
return getReplSrcTxnIds();
+ case TXN_TYPE:
+ return getTxn_type();
+
}
throw new IllegalStateException();
}
@@ -460,6 +520,8 @@ import org.slf4j.LoggerFactory;
return isSetReplPolicy();
case REPL_SRC_TXN_IDS:
return isSetReplSrcTxnIds();
+ case TXN_TYPE:
+ return isSetTxn_type();
}
throw new IllegalStateException();
}
@@ -531,6 +593,15 @@ import org.slf4j.LoggerFactory;
return false;
}
+ boolean this_present_txn_type = true && this.isSetTxn_type();
+ boolean that_present_txn_type = true && that.isSetTxn_type();
+ if (this_present_txn_type || that_present_txn_type) {
+ if (!(this_present_txn_type && that_present_txn_type))
+ return false;
+ if (!this.txn_type.equals(that.txn_type))
+ return false;
+ }
+
return true;
}
@@ -568,6 +639,11 @@ import org.slf4j.LoggerFactory;
if (present_replSrcTxnIds)
list.add(replSrcTxnIds);
+ boolean present_txn_type = true && (isSetTxn_type());
+ list.add(present_txn_type);
+ if (present_txn_type)
+ list.add(txn_type.getValue());
+
return list.hashCode();
}
@@ -639,6 +715,16 @@ import org.slf4j.LoggerFactory;
return lastComparison;
}
}
+ lastComparison = Boolean.valueOf(isSetTxn_type()).compareTo(other.isSetTxn_type());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTxn_type()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.txn_type, other.txn_type);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -708,6 +794,16 @@ import org.slf4j.LoggerFactory;
}
first = false;
}
+ if (isSetTxn_type()) {
+ if (!first) sb.append(", ");
+ sb.append("txn_type:");
+ if (this.txn_type == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.txn_type);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -823,6 +919,14 @@ import org.slf4j.LoggerFactory;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 7: // TXN_TYPE
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.findByValue(iprot.readI32());
+ struct.setTxn_typeIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -877,6 +981,13 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldEnd();
}
}
+ if (struct.txn_type != null) {
+ if (struct.isSetTxn_type()) {
+ oprot.writeFieldBegin(TXN_TYPE_FIELD_DESC);
+ oprot.writeI32(struct.txn_type.getValue());
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -907,7 +1018,10 @@ import org.slf4j.LoggerFactory;
if (struct.isSetReplSrcTxnIds()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetTxn_type()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetAgentInfo()) {
oprot.writeString(struct.agentInfo);
}
@@ -923,6 +1037,9 @@ import org.slf4j.LoggerFactory;
}
}
}
+ if (struct.isSetTxn_type()) {
+ oprot.writeI32(struct.txn_type.getValue());
+ }
}
@Override
@@ -934,7 +1051,7 @@ import org.slf4j.LoggerFactory;
struct.setUserIsSet(true);
struct.hostname = iprot.readString();
struct.setHostnameIsSet(true);
- BitSet incoming = iprot.readBitSet(3);
+ BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.agentInfo = iprot.readString();
struct.setAgentInfoIsSet(true);
@@ -956,6 +1073,10 @@ import org.slf4j.LoggerFactory;
}
struct.setReplSrcTxnIdsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.findByValue(iprot.readI32());
+ struct.setTxn_typeIsSet(true);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
new file mode 100644
index 0000000..349f37f
--- /dev/null
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java
@@ -0,0 +1,51 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+
+import java.util.Map;
+import java.util.HashMap;
+import org.apache.thrift.TEnum;
+
+public enum TxnType implements org.apache.thrift.TEnum {
+ DEFAULT(0),
+ REPL_CREATED(1),
+ READ_ONLY(2),
+ COMPACTION(3);
+
+ private final int value;
+
+ private TxnType(int value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the integer value of this enum value, as defined in the Thrift IDL.
+ */
+ public int getValue() {
+ return value;
+ }
+
+ /**
+ * Find a the enum type by its integer value, as defined in the Thrift IDL.
+ * @return null if the value is not found.
+ */
+ public static TxnType findByValue(int value) {
+ switch (value) {
+ case 0:
+ return DEFAULT;
+ case 1:
+ return REPL_CREATED;
+ case 2:
+ return READ_ONLY;
+ case 3:
+ return COMPACTION;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
index 2953a8f..13e287e 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php
@@ -221,6 +221,19 @@ final class ResourceType {
);
}
+final class TxnType {
+ const DEFAULT = 0;
+ const REPL_CREATED = 1;
+ const READ_ONLY = 2;
+ const COMPACTION = 3;
+ static public $__names = array(
+ 0 => 'DEFAULT',
+ 1 => 'REPL_CREATED',
+ 2 => 'READ_ONLY',
+ 3 => 'COMPACTION',
+ );
+}
+
final class FileMetadataExprType {
const ORC_SARG = 1;
static public $__names = array(
@@ -16778,6 +16791,10 @@ class OpenTxnRequest {
* @var int[]
*/
public $replSrcTxnIds = null;
+ /**
+ * @var int
+ */
+ public $txn_type = 0;
public function __construct($vals=null) {
if (!isset(self::$_TSPEC)) {
@@ -16810,6 +16827,10 @@ class OpenTxnRequest {
'type' => TType::I64,
),
),
+ 7 => array(
+ 'var' => 'txn_type',
+ 'type' => TType::I32,
+ ),
);
}
if (is_array($vals)) {
@@ -16831,6 +16852,9 @@ class OpenTxnRequest {
if (isset($vals['replSrcTxnIds'])) {
$this->replSrcTxnIds = $vals['replSrcTxnIds'];
}
+ if (isset($vals['txn_type'])) {
+ $this->txn_type = $vals['txn_type'];
+ }
}
}
@@ -16905,6 +16929,13 @@ class OpenTxnRequest {
$xfer += $input->skip($ftype);
}
break;
+ case 7:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->txn_type);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -16960,6 +16991,11 @@ class OpenTxnRequest {
}
$xfer += $output->writeFieldEnd();
}
+ if ($this->txn_type !== null) {
+ $xfer += $output->writeFieldBegin('txn_type', TType::I32, 7);
+ $xfer += $output->writeI32($this->txn_type);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index f8b8628..8f149d1 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -333,6 +333,26 @@ class ResourceType:
"ARCHIVE": 3,
}
+class TxnType:
+ DEFAULT = 0
+ REPL_CREATED = 1
+ READ_ONLY = 2
+ COMPACTION = 3
+
+ _VALUES_TO_NAMES = {
+ 0: "DEFAULT",
+ 1: "REPL_CREATED",
+ 2: "READ_ONLY",
+ 3: "COMPACTION",
+ }
+
+ _NAMES_TO_VALUES = {
+ "DEFAULT": 0,
+ "REPL_CREATED": 1,
+ "READ_ONLY": 2,
+ "COMPACTION": 3,
+ }
+
class FileMetadataExprType:
ORC_SARG = 1
@@ -11672,6 +11692,7 @@ class OpenTxnRequest:
- agentInfo
- replPolicy
- replSrcTxnIds
+ - txn_type
"""
thrift_spec = (
@@ -11682,15 +11703,17 @@ class OpenTxnRequest:
(4, TType.STRING, 'agentInfo', None, "Unknown", ), # 4
(5, TType.STRING, 'replPolicy', None, None, ), # 5
(6, TType.LIST, 'replSrcTxnIds', (TType.I64,None), None, ), # 6
+ (7, TType.I32, 'txn_type', None, 0, ), # 7
)
- def __init__(self, num_txns=None, user=None, hostname=None, agentInfo=thrift_spec[4][4], replPolicy=None, replSrcTxnIds=None,):
+ def __init__(self, num_txns=None, user=None, hostname=None, agentInfo=thrift_spec[4][4], replPolicy=None, replSrcTxnIds=None, txn_type=thrift_spec[7][4],):
self.num_txns = num_txns
self.user = user
self.hostname = hostname
self.agentInfo = agentInfo
self.replPolicy = replPolicy
self.replSrcTxnIds = replSrcTxnIds
+ self.txn_type = txn_type
def read(self, iprot):
if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -11736,6 +11759,11 @@ class OpenTxnRequest:
iprot.readListEnd()
else:
iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.I32:
+ self.txn_type = iprot.readI32()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -11773,6 +11801,10 @@ class OpenTxnRequest:
oprot.writeI64(iter515)
oprot.writeListEnd()
oprot.writeFieldEnd()
+ if self.txn_type is not None:
+ oprot.writeFieldBegin('txn_type', TType.I32, 7)
+ oprot.writeI32(self.txn_type)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -11794,6 +11826,7 @@ class OpenTxnRequest:
value = (value * 31) ^ hash(self.agentInfo)
value = (value * 31) ^ hash(self.replPolicy)
value = (value * 31) ^ hash(self.replSrcTxnIds)
+ value = (value * 31) ^ hash(self.txn_type)
return value
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index dfc5d7b..9e5f086 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -155,6 +155,15 @@ module ResourceType
VALID_VALUES = Set.new([JAR, FILE, ARCHIVE]).freeze
end
+module TxnType
+ DEFAULT = 0
+ REPL_CREATED = 1
+ READ_ONLY = 2
+ COMPACTION = 3
+ VALUE_MAP = {0 => "DEFAULT", 1 => "REPL_CREATED", 2 => "READ_ONLY", 3 => "COMPACTION"}
+ VALID_VALUES = Set.new([DEFAULT, REPL_CREATED, READ_ONLY, COMPACTION]).freeze
+end
+
module FileMetadataExprType
ORC_SARG = 1
VALUE_MAP = {1 => "ORC_SARG"}
@@ -2604,6 +2613,7 @@ class OpenTxnRequest
AGENTINFO = 4
REPLPOLICY = 5
REPLSRCTXNIDS = 6
+ TXN_TYPE = 7
FIELDS = {
NUM_TXNS => {:type => ::Thrift::Types::I32, :name => 'num_txns'},
@@ -2611,7 +2621,8 @@ class OpenTxnRequest
HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true},
REPLPOLICY => {:type => ::Thrift::Types::STRING, :name => 'replPolicy', :optional => true},
- REPLSRCTXNIDS => {:type => ::Thrift::Types::LIST, :name => 'replSrcTxnIds', :element => {:type => ::Thrift::Types::I64}, :optional => true}
+ REPLSRCTXNIDS => {:type => ::Thrift::Types::LIST, :name => 'replSrcTxnIds', :element => {:type => ::Thrift::Types::I64}, :optional => true},
+ TXN_TYPE => {:type => ::Thrift::Types::I32, :name => 'txn_type', :default => 0, :optional => true, :enum_class => ::TxnType}
}
def struct_fields; FIELDS; end
@@ -2620,6 +2631,9 @@ class OpenTxnRequest
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field num_txns is unset!') unless @num_txns
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+ unless @txn_type.nil? || ::TxnType::VALID_VALUES.include?(@txn_type)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field txn_type!')
+ end
end
::Thrift::Struct.generate_accessors self
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 748b56b..3c81f93 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2794,7 +2794,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public long openTxn(String user) throws TException {
- OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null);
+ OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, null);
+ return txns.getTxn_ids().get(0);
+ }
+
+ @Override
+ public long openTxn(String user, TxnType txnType) throws TException {
+ OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, txnType);
return txns.getTxn_ids().get(0);
}
@@ -2802,17 +2808,17 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public List<Long> replOpenTxn(String replPolicy, List<Long> srcTxnIds, String user) throws TException {
// As this is called from replication task, the user is the user who has fired the repl command.
// This is required for standalone metastore authentication.
- OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds != null ? srcTxnIds.size() : 1, replPolicy, srcTxnIds);
+ OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds != null ? srcTxnIds.size() : 1, replPolicy, srcTxnIds, null);
return txns.getTxn_ids();
}
@Override
public OpenTxnsResponse openTxns(String user, int numTxns) throws TException {
- return openTxnsIntr(user, numTxns, null, null);
+ return openTxnsIntr(user, numTxns, null, null, null);
}
private OpenTxnsResponse openTxnsIntr(String user, int numTxns, String replPolicy,
- List<Long> srcTxnIds) throws TException {
+ List<Long> srcTxnIds, TxnType txnType) throws TException {
String hostname;
try {
hostname = InetAddress.getLocalHost().getHostName();
@@ -2830,6 +2836,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
} else {
assert srcTxnIds == null;
}
+ if(txnType != null) {
+ rqst.setTxn_type(txnType);
+ }
return client.open_txns(rqst);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index c96bf0d..158740d 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -116,6 +116,7 @@ import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
import org.apache.hadoop.hive.metastore.api.TxnOpenException;
import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
+import org.apache.hadoop.hive.metastore.api.TxnType;
import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
@@ -2856,6 +2857,15 @@ public interface IMetaStoreClient {
long openTxn(String user) throws TException;
/**
+ * Initiate a transaction with given type.
+ * @param user User who is opening this transaction.
+ * @param txnType Type of needed transaction.
+ * @return transaction identifier
+ * @throws TException
+ */
+ long openTxn(String user, TxnType txnType) throws TException;
+
+ /**
* Initiate a transaction at the target cluster.
* @param replPolicy The replication policy to uniquely identify the source cluster.
* @param srcTxnIds The list of transaction ids at the source cluster
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index 7d09a5c..9576f87 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -833,6 +833,13 @@ enum ResourceType {
ARCHIVE = 3,
}
+enum TxnType {
+ DEFAULT = 0,
+ REPL_CREATED = 1,
+ READ_ONLY = 2,
+ COMPACTION = 3
+}
+
struct ResourceUri {
1: ResourceType resourceType,
2: string uri,
@@ -883,6 +890,7 @@ struct OpenTxnRequest {
4: optional string agentInfo = "Unknown",
5: optional string replPolicy,
6: optional list<i64> replSrcTxnIds,
+ 7: optional TxnType txn_type = TxnType.DEFAULT,
}
struct OpenTxnsResponse {
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 2a62903..91a9ab4 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -166,19 +166,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
//todo: make these like OperationType and remove above char constants
enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN}
- public enum TxnType {
- DEFAULT(0), REPL_CREATED(1), READ_ONLY(2);
-
- private final int value;
- TxnType(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
- }
-
// Lock states
static final protected char LOCK_ACQUIRED = 'a';
static final protected char LOCK_WAITING = 'w';
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index a3f5e2c..898a94d 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -2227,20 +2227,26 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public long openTxn(String user, TxnType txnType) throws TException {
+ OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, txnType);
+ return txns.getTxn_ids().get(0);
+ }
+
+ @Override
public OpenTxnsResponse openTxns(String user, int numTxns) throws TException {
- return openTxnsIntr(user, numTxns, null, null);
+ return openTxnsIntr(user, numTxns, null, null, null);
}
@Override
public List<Long> replOpenTxn(String replPolicy, List<Long> srcTxnIds, String user) throws TException {
// As this is called from replication task, the user is the user who has fired the repl command.
// This is required for standalone metastore authentication.
- OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds.size(), replPolicy, srcTxnIds);
+ OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds.size(), replPolicy, srcTxnIds, null);
return txns.getTxn_ids();
}
private OpenTxnsResponse openTxnsIntr(String user, int numTxns, String replPolicy,
- List<Long> srcTxnIds) throws TException {
+ List<Long> srcTxnIds, TxnType txnType) throws TException {
String hostname;
try {
hostname = InetAddress.getLocalHost().getHostName();
@@ -2258,6 +2264,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
} else {
assert srcTxnIds == null;
}
+ if (txnType != null) {
+ rqst.setTxn_type(txnType);
+ }
return client.open_txns(rqst);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index 2a70ec3..4cdc035 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.LockResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnType;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -298,6 +299,14 @@ public class TestHiveMetaStoreTxns {
Assert.assertTrue(sawFive);
}
+ @Test
+ public void testOpenTxnWithType() throws Exception {
+ long txnId = client.openTxn("me", TxnType.DEFAULT);
+ client.commitTxn(txnId);
+ ValidTxnList validTxns = client.getValidTxns();
+ Assert.assertTrue(validTxns.isTxnValid(txnId));
+ }
+
@Before
public void setUp() throws Exception {
MetaStoreTestUtils.setConfForStandloneMode(conf);