You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by bh...@apache.org on 2019/04/01 20:13:10 UTC
[hive] branch master updated (470e966 -> 7af186a)
This is an automated email from the ASF dual-hosted git repository.
bharos92 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from 470e966 HIVE-21396 : TestCliDriver#vector_groupby_reduce is flaky - rounding error (Vihang Karajgaonkar reviewed by Laszlo Bodor)
new 50cdbe2 HIVE-21205 : Tests for replace flag in insert event messages in Metastore notifications (Bharath Krishna reviewed by Vihang karajgaonkar)
new 7af186a HIVE-21526 : JSONDropDatabaseMessage needs to have the full database object (Bharath Krishna reviewed by Vihang Karajgaonkar)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../listener/TestDbNotificationListener.java | 26 +++++++++++++++++-----
.../metastore/messaging/DropDatabaseMessage.java | 4 ++++
.../hive/metastore/messaging/MessageBuilder.java | 2 +-
.../messaging/json/JSONDropDatabaseMessage.java | 19 +++++++++++++---
4 files changed, 41 insertions(+), 10 deletions(-)
[hive] 02/02: HIVE-21526 : JSONDropDatabaseMessage needs to have
the full database object (Bharath Krishna reviewed by Vihang Karajgaonkar)
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharos92 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 7af186a6c134e83a4ca3dc705035d635df574b7f
Author: Bharath Krishna <bh...@cloudera.com>
AuthorDate: Wed Mar 27 11:12:50 2019 -0700
HIVE-21526 : JSONDropDatabaseMessage needs to have the full database object (Bharath Krishna reviewed by Vihang Karajgaonkar)
---
.../hcatalog/listener/TestDbNotificationListener.java | 11 +++++++----
.../hive/metastore/messaging/DropDatabaseMessage.java | 4 ++++
.../hive/metastore/messaging/MessageBuilder.java | 2 +-
.../messaging/json/JSONDropDatabaseMessage.java | 19 ++++++++++++++++---
4 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index a2f9f3f..7180564 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -406,8 +406,11 @@ public class TestDbNotificationListener {
String dbName2 = "dropdb2";
String dbLocationUri = testTempDir;
String dbDescription = "no description";
- Database db = new Database(dbName, dbDescription, dbLocationUri, emptyParameters);
- msClient.createDatabase(db);
+ msClient.createDatabase(new Database(dbName, dbDescription, dbLocationUri, emptyParameters));
+
+ // Get the DB for comparison below since it may include additional parameters
+ Database db = msClient.getDatabase(dbName);
+ // Drop the database
msClient.dropDatabase(dbName);
// Read notification from metastore
@@ -428,6 +431,7 @@ public class TestDbNotificationListener {
// Parse the message field
DropDatabaseMessage dropDbMsg = md.getDropDatabaseMessage(event.getMessage());
assertEquals(dbName, dropDbMsg.getDB());
+ assertEquals(db, dropDbMsg.getDatabaseObject());
// Verify the eventID was passed to the non-transactional listener
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_DATABASE, firstEventId + 2);
@@ -435,8 +439,7 @@ public class TestDbNotificationListener {
// When hive.metastore.transactional.event.listeners is set,
// a failed event should not create a new notification
- db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters);
- msClient.createDatabase(db);
+ msClient.createDatabase(new Database(dbName2, dbDescription, dbLocationUri, emptyParameters));
DummyRawStoreFailEvent.setEventSucceed(false);
try {
msClient.dropDatabase(dbName2);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java
index a450d47..f8a567d 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java
@@ -19,9 +19,13 @@
package org.apache.hadoop.hive.metastore.messaging;
+import org.apache.hadoop.hive.metastore.api.Database;
+
public abstract class DropDatabaseMessage extends EventMessage {
protected DropDatabaseMessage() {
super(EventType.DROP_DATABASE);
}
+
+ public abstract Database getDatabaseObject() throws Exception;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java
index 15c4769..aa83da4 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java
@@ -175,7 +175,7 @@ public class MessageBuilder {
}
public DropDatabaseMessage buildDropDatabaseMessage(Database db) {
- return new JSONDropDatabaseMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, db.getName(), now());
+ return new JSONDropDatabaseMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, db, now());
}
public CreateTableMessage buildCreateTableMessage(Table table, Iterator<String> fileIter) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
index d2a75bf..ae79161 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
@@ -19,9 +19,12 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.hive.metastore.messaging.MessageBuilder;
+import org.apache.thrift.TException;
/**
* JSON implementation of DropDatabaseMessage.
@@ -29,7 +32,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
public class JSONDropDatabaseMessage extends DropDatabaseMessage {
@JsonProperty
- String server, servicePrincipal, db;
+ String server, servicePrincipal, db, dbJson;
@JsonProperty
Long timestamp;
@@ -39,11 +42,16 @@ public class JSONDropDatabaseMessage extends DropDatabaseMessage {
*/
public JSONDropDatabaseMessage() {}
- public JSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp) {
+ public JSONDropDatabaseMessage(String server, String servicePrincipal, Database db, Long timestamp) {
this.server = server;
this.servicePrincipal = servicePrincipal;
- this.db = db;
+ this.db = db.getName();
this.timestamp = timestamp;
+ try {
+ this.dbJson = MessageBuilder.createDatabaseObjJson(db);
+ } catch (TException ex) {
+ throw new IllegalArgumentException("Could not serialize database object", ex);
+ }
checkValid();
}
@@ -61,6 +69,11 @@ public class JSONDropDatabaseMessage extends DropDatabaseMessage {
public Long getTimestamp() { return timestamp; }
@Override
+ public Database getDatabaseObject() throws Exception {
+ return (Database) MessageBuilder.getTObj(dbJson, Database.class);
+ }
+
+ @Override
public String toString() {
try {
return JSONMessageDeserializer.mapper.writeValueAsString(this);
[hive] 01/02: HIVE-21205 : Tests for replace flag in insert event
messages in Metastore notifications (Bharath Krishna reviewed by Vihang
karajgaonkar)
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharos92 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 50cdbe24fe5ec026023ea585589ee51460e79ffb
Author: Bharath Krishna <bh...@cloudera.com>
AuthorDate: Tue Feb 5 16:12:01 2019 -0800
HIVE-21205 : Tests for replace flag in insert event messages in Metastore notifications (Bharath Krishna reviewed by Vihang karajgaonkar)
---
.../hcatalog/listener/TestDbNotificationListener.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 0c3e0466..a2f9f3f 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -21,6 +21,7 @@ package org.apache.hive.hcatalog.listener;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.util.concurrent.TimeUnit;
@@ -1080,6 +1081,7 @@ public class TestDbNotificationListener {
data.setInsertData(insertData);
insertData.addToFilesAdded(fileAdded);
insertData.addToFilesAddedChecksum(checksumAdded);
+ insertData.setReplace(false);
FireEventRequest rqst = new FireEventRequest(true, data);
rqst.setDbName(defaultDbName);
rqst.setTableName(tblName);
@@ -1103,6 +1105,7 @@ public class TestDbNotificationListener {
assertEquals(defaultDbName, insertMessage.getDB());
assertEquals(tblName, insertMessage.getTable());
assertEquals(TableType.MANAGED_TABLE.toString(), insertMessage.getTableType());
+ assertFalse(insertMessage.isReplace());
// Verify the eventID was passed to the non-transactional listener
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2);
@@ -1147,6 +1150,7 @@ public class TestDbNotificationListener {
data.setInsertData(insertData);
insertData.addToFilesAdded(fileAdded);
insertData.addToFilesAddedChecksum(checksumAdded);
+ insertData.setReplace(false);
FireEventRequest rqst = new FireEventRequest(true, data);
rqst.setDbName(defaultDbName);
rqst.setTableName(tblName);
@@ -1167,7 +1171,7 @@ public class TestDbNotificationListener {
verifyInsert(event, defaultDbName, tblName);
InsertMessage insertMessage = md.getInsertMessage(event.getMessage());
List<String> ptnValues = insertMessage.getPtnObj().getValues();
-
+ assertFalse(insertMessage.isReplace());
assertEquals(partKeyVals, ptnValues);
// Verify the eventID was passed to the non-transactional listener
@@ -1261,6 +1265,8 @@ public class TestDbNotificationListener {
assertEquals(EventType.INSERT.toString(), event.getEventType());
// Parse the message field
verifyInsert(event, defaultDbName, tblName);
+ InsertMessage insertMsg = md.getInsertMessage(event.getMessage());
+ assertFalse(insertMsg.isReplace());
event = rsp.getEvents().get(5);
assertEquals(firstEventId + 6, event.getEventId());
@@ -1387,7 +1393,9 @@ public class TestDbNotificationListener {
assertEquals(EventType.INSERT.toString(), event.getEventType());
// Parse the message field
verifyInsert(event, null, tblName);
-
+ // Verify the replace flag.
+ InsertMessage insertMsg = md.getInsertMessage(event.getMessage());
+ assertFalse(insertMsg.isReplace());
event = rsp.getEvents().get(8);
assertEquals(firstEventId + 9, event.getEventId());
assertEquals(EventType.INSERT.toString(), event.getEventType());
@@ -1433,6 +1441,9 @@ public class TestDbNotificationListener {
event = rsp.getEvents().get(28);
assertEquals(firstEventId + 29, event.getEventId());
assertEquals(EventType.INSERT.toString(), event.getEventType());
+ // Verify the replace flag.
+ insertMsg = md.getInsertMessage(event.getMessage());
+ assertTrue(insertMsg.isReplace());
// replace-overwrite introduces no new files
assertTrue(event.getMessage().matches(".*\"files\":\\[\\].*"));