You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2020/03/31 07:49:56 UTC
[hive] 01/02: HIVE-21851: FireEventResponse should include event id
when available (Vihang Karajgaonkar, reviewed by Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
vihangk1 pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 7426703e60112937ed3749526fcfbfeda2bf2ace
Author: Vihang Karajgaonkar <vi...@apache.org>
AuthorDate: Mon Mar 30 16:48:08 2020 -0700
HIVE-21851: FireEventResponse should include event id when available (Vihang Karajgaonkar, reviewed by Naveen Gangam)
---
.../listener/TestDbNotificationListener.java | 12 ++-
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 34 ++++++-
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 15 ++-
.../hive/metastore/api/FireEventResponse.java | 112 ++++++++++++++++++++-
.../src/gen/thrift/gen-php/metastore/Types.php | 27 ++++-
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 19 ++++
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 3 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 9 +-
.../src/main/thrift/hive_metastore.thrift | 2 +-
9 files changed, 219 insertions(+), 14 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 37b2bd8..bfb1982 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.FireEventRequest;
import org.apache.hadoop.hive.metastore.api.FireEventRequestData;
+import org.apache.hadoop.hive.metastore.api.FireEventResponse;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.FunctionType;
import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
@@ -98,6 +99,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule;
import org.apache.hive.hcatalog.data.Pair;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -1047,7 +1049,10 @@ public class TestDbNotificationListener {
rqst.setDbName(defaultDbName);
rqst.setTableName(tblName);
// Event 2
- msClient.fireListenerEvent(rqst);
+ FireEventResponse response = msClient.fireListenerEvent(rqst);
+ assertTrue("Event id must be set in the fireEvent response", response.isSetEventId());
+ Assert.assertNotNull(response.getEventId());
+ Assert.assertTrue(response.getEventId() != -1);
// Get notifications from metastore
NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
@@ -1116,7 +1121,10 @@ public class TestDbNotificationListener {
rqst.setTableName(tblName);
rqst.setPartitionVals(partCol1Vals);
// Event 3
- msClient.fireListenerEvent(rqst);
+ FireEventResponse response = msClient.fireListenerEvent(rqst);
+ assertTrue("Event id must be set in the fireEvent response", response.isSetEventId());
+ Assert.assertNotNull(response.getEventId());
+ Assert.assertTrue(response.getEventId() != -1);
// Get notifications from metastore
NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 5f2948e..f3e5d3a 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -22756,6 +22756,10 @@ FireEventResponse::~FireEventResponse() throw() {
}
+void FireEventResponse::__set_eventId(const int64_t val) {
+ this->eventId = val;
+}
+
uint32_t FireEventResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
@@ -22775,7 +22779,20 @@ uint32_t FireEventResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_STOP) {
break;
}
- xfer += iprot->skip(ftype);
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I64) {
+ xfer += iprot->readI64(this->eventId);
+ this->__isset.eventId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
xfer += iprot->readFieldEnd();
}
@@ -22789,6 +22806,10 @@ uint32_t FireEventResponse::write(::apache::thrift::protocol::TProtocol* oprot)
apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
xfer += oprot->writeStructBegin("FireEventResponse");
+ xfer += oprot->writeFieldBegin("eventId", ::apache::thrift::protocol::T_I64, 1);
+ xfer += oprot->writeI64(this->eventId);
+ xfer += oprot->writeFieldEnd();
+
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -22796,20 +22817,23 @@ uint32_t FireEventResponse::write(::apache::thrift::protocol::TProtocol* oprot)
void swap(FireEventResponse &a, FireEventResponse &b) {
using ::std::swap;
- (void) a;
- (void) b;
+ swap(a.eventId, b.eventId);
+ swap(a.__isset, b.__isset);
}
FireEventResponse::FireEventResponse(const FireEventResponse& other887) {
- (void) other887;
+ eventId = other887.eventId;
+ __isset = other887.__isset;
}
FireEventResponse& FireEventResponse::operator=(const FireEventResponse& other888) {
- (void) other888;
+ eventId = other888.eventId;
+ __isset = other888.__isset;
return *this;
}
void FireEventResponse::printTo(std::ostream& out) const {
using ::apache::thrift::to_string;
out << "FireEventResponse(";
+ out << "eventId=" << to_string(eventId);
out << ")";
}
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index fb86c97..a3a28ed 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -9353,19 +9353,30 @@ inline std::ostream& operator<<(std::ostream& out, const FireEventRequest& obj)
return out;
}
+typedef struct _FireEventResponse__isset {
+ _FireEventResponse__isset() : eventId(false) {}
+ bool eventId :1;
+} _FireEventResponse__isset;
class FireEventResponse {
public:
FireEventResponse(const FireEventResponse&);
FireEventResponse& operator=(const FireEventResponse&);
- FireEventResponse() {
+ FireEventResponse() : eventId(0) {
}
virtual ~FireEventResponse() throw();
+ int64_t eventId;
+
+ _FireEventResponse__isset __isset;
+
+ void __set_eventId(const int64_t val);
- bool operator == (const FireEventResponse & /* rhs */) const
+ bool operator == (const FireEventResponse & rhs) const
{
+ if (!(eventId == rhs.eventId))
+ return false;
return true;
}
bool operator != (const FireEventResponse &rhs) const {
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java
index 9125d86..1b48ecb 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventResponse.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class FireEventResponse implements org.apache.thrift.TBase<FireEventResponse, FireEventResponse._Fields>, java.io.Serializable, Cloneable, Comparable<FireEventResponse> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("FireEventResponse");
+ private static final org.apache.thrift.protocol.TField EVENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("eventId", org.apache.thrift.protocol.TType.I64, (short)1);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -45,10 +46,11 @@ import org.slf4j.LoggerFactory;
schemes.put(TupleScheme.class, new FireEventResponseTupleSchemeFactory());
}
+ private long eventId; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
+ EVENT_ID((short)1, "eventId");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -63,6 +65,8 @@ import org.slf4j.LoggerFactory;
*/
public static _Fields findByThriftId(int fieldId) {
switch(fieldId) {
+ case 1: // EVENT_ID
+ return EVENT_ID;
default:
return null;
}
@@ -101,9 +105,15 @@ import org.slf4j.LoggerFactory;
return _fieldName;
}
}
+
+ // isset id assignments
+ private static final int __EVENTID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.EVENT_ID, new org.apache.thrift.meta_data.FieldMetaData("eventId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(FireEventResponse.class, metaDataMap);
}
@@ -111,10 +121,20 @@ import org.slf4j.LoggerFactory;
public FireEventResponse() {
}
+ public FireEventResponse(
+ long eventId)
+ {
+ this();
+ this.eventId = eventId;
+ setEventIdIsSet(true);
+ }
+
/**
* Performs a deep copy on <i>other</i>.
*/
public FireEventResponse(FireEventResponse other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.eventId = other.eventId;
}
public FireEventResponse deepCopy() {
@@ -123,15 +143,50 @@ import org.slf4j.LoggerFactory;
@Override
public void clear() {
+ setEventIdIsSet(false);
+ this.eventId = 0;
+ }
+
+ public long getEventId() {
+ return this.eventId;
+ }
+
+ public void setEventId(long eventId) {
+ this.eventId = eventId;
+ setEventIdIsSet(true);
+ }
+
+ public void unsetEventId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __EVENTID_ISSET_ID);
+ }
+
+ /** Returns true if field eventId is set (has been assigned a value) and false otherwise */
+ public boolean isSetEventId() {
+ return EncodingUtils.testBit(__isset_bitfield, __EVENTID_ISSET_ID);
+ }
+
+ public void setEventIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __EVENTID_ISSET_ID, value);
}
public void setFieldValue(_Fields field, Object value) {
switch (field) {
+ case EVENT_ID:
+ if (value == null) {
+ unsetEventId();
+ } else {
+ setEventId((Long)value);
+ }
+ break;
+
}
}
public Object getFieldValue(_Fields field) {
switch (field) {
+ case EVENT_ID:
+ return getEventId();
+
}
throw new IllegalStateException();
}
@@ -143,6 +198,8 @@ import org.slf4j.LoggerFactory;
}
switch (field) {
+ case EVENT_ID:
+ return isSetEventId();
}
throw new IllegalStateException();
}
@@ -160,6 +217,15 @@ import org.slf4j.LoggerFactory;
if (that == null)
return false;
+ boolean this_present_eventId = true;
+ boolean that_present_eventId = true;
+ if (this_present_eventId || that_present_eventId) {
+ if (!(this_present_eventId && that_present_eventId))
+ return false;
+ if (this.eventId != that.eventId)
+ return false;
+ }
+
return true;
}
@@ -167,6 +233,11 @@ import org.slf4j.LoggerFactory;
public int hashCode() {
List<Object> list = new ArrayList<Object>();
+ boolean present_eventId = true;
+ list.add(present_eventId);
+ if (present_eventId)
+ list.add(eventId);
+
return list.hashCode();
}
@@ -178,6 +249,16 @@ import org.slf4j.LoggerFactory;
int lastComparison = 0;
+ lastComparison = Boolean.valueOf(isSetEventId()).compareTo(other.isSetEventId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetEventId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.eventId, other.eventId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -198,6 +279,9 @@ import org.slf4j.LoggerFactory;
StringBuilder sb = new StringBuilder("FireEventResponse(");
boolean first = true;
+ sb.append("eventId:");
+ sb.append(this.eventId);
+ first = false;
sb.append(")");
return sb.toString();
}
@@ -217,6 +301,8 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -241,6 +327,14 @@ import org.slf4j.LoggerFactory;
break;
}
switch (schemeField.id) {
+ case 1: // EVENT_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.eventId = iprot.readI64();
+ struct.setEventIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -254,6 +348,9 @@ import org.slf4j.LoggerFactory;
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(EVENT_ID_FIELD_DESC);
+ oprot.writeI64(struct.eventId);
+ oprot.writeFieldEnd();
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -271,11 +368,24 @@ import org.slf4j.LoggerFactory;
@Override
public void write(org.apache.thrift.protocol.TProtocol prot, FireEventResponse struct) throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetEventId()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetEventId()) {
+ oprot.writeI64(struct.eventId);
+ }
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, FireEventResponse struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.eventId = iprot.readI64();
+ struct.setEventIdIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index 688574e..00641e5 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -22480,12 +22480,25 @@ class FireEventRequest {
class FireEventResponse {
static $_TSPEC;
+ /**
+ * @var int
+ */
+ public $eventId = null;
- public function __construct() {
+ public function __construct($vals=null) {
if (!isset(self::$_TSPEC)) {
self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'eventId',
+ 'type' => TType::I64,
+ ),
);
}
+ if (is_array($vals)) {
+ if (isset($vals['eventId'])) {
+ $this->eventId = $vals['eventId'];
+ }
+ }
}
public function getName() {
@@ -22507,6 +22520,13 @@ class FireEventResponse {
}
switch ($fid)
{
+ case 1:
+ if ($ftype == TType::I64) {
+ $xfer += $input->readI64($this->eventId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -22520,6 +22540,11 @@ class FireEventResponse {
public function write($output) {
$xfer = 0;
$xfer += $output->writeStructBegin('FireEventResponse');
+ if ($this->eventId !== null) {
+ $xfer += $output->writeFieldBegin('eventId', TType::I64, 1);
+ $xfer += $output->writeI64($this->eventId);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index e429718..3f12407 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -15734,10 +15734,19 @@ class FireEventRequest:
return not (self == other)
class FireEventResponse:
+ """
+ Attributes:
+ - eventId
+ """
thrift_spec = (
+ None, # 0
+ (1, TType.I64, 'eventId', None, None, ), # 1
)
+ def __init__(self, eventId=None,):
+ self.eventId = eventId
+
def read(self, iprot):
if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
@@ -15747,6 +15756,11 @@ class FireEventResponse:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
+ if fid == 1:
+ if ftype == TType.I64:
+ self.eventId = iprot.readI64()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -15757,6 +15771,10 @@ class FireEventResponse:
oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
return
oprot.writeStructBegin('FireEventResponse')
+ if self.eventId is not None:
+ oprot.writeFieldBegin('eventId', TType.I64, 1)
+ oprot.writeI64(self.eventId)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -15766,6 +15784,7 @@ class FireEventResponse:
def __hash__(self):
value = 17
+ value = (value * 31) ^ hash(self.eventId)
return value
def __repr__(self):
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 674a94e..7cbca4b 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -3506,9 +3506,10 @@ end
class FireEventResponse
include ::Thrift::Struct, ::Thrift::Struct_Union
+ EVENTID = 1
FIELDS = {
-
+ EVENTID => {:type => ::Thrift::Types::I64, :name => 'eventId'}
}
def struct_fields; FIELDS; end
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index d7c8050..e07e225 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -7649,7 +7649,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.INSERT, event);
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.INSERT, event);
- return new FireEventResponse();
+ FireEventResponse response = new FireEventResponse();
+ if (event.getParameters() != null && event.getParameters()
+ .containsKey(
+ MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) {
+ response.setEventId(Long.valueOf(event.getParameters()
+ .get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)));
+ }
+ return response;
default:
throw new TException("Event type " + rqst.getData().getSetField().toString()
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index b3ffcb7..fcd1e56 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -1138,7 +1138,7 @@ struct FireEventRequest {
}
struct FireEventResponse {
- // NOP for now, this is just a place holder for future responses
+ 1: i64 eventId
}
struct WriteNotificationLogRequest {